serving_chat.py 81.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
22
23
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
24
25
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.parser.harmony_utils import (
26
27
28
29
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
30
    parse_chat_inputs_to_harmony_messages,
31
32
33
    parse_chat_output,
    render_for_completion,
)
34
from vllm.entrypoints.openai.protocol import (
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
54
55
56
57
58
from vllm.entrypoints.openai.serving_engine import (
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
59
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
60
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
61
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
62
from vllm.inputs.data import TokensPrompt
63
from vllm.logger import init_logger
64
from vllm.logprobs import Logprob
65
from vllm.outputs import CompletionOutput, RequestOutput
66
from vllm.sampling_params import BeamSearchParams, SamplingParams
67
68
69
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
    MistralTokenizer,
70
71
72
73
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
74
75
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
76
from vllm.utils.collection_utils import as_list
77
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
78
79
80
81
82

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
83
84
85
    def __init__(
        self,
        engine_client: EngineClient,
86
        models: OpenAIServingModels,
87
88
        response_role: str,
        *,
89
90
        request_logger: RequestLogger | None,
        chat_template: str | None,
91
        chat_template_content_format: ChatTemplateContentFormatOption,
92
        trust_request_chat_template: bool = False,
93
        return_tokens_as_token_ids: bool = False,
94
        reasoning_parser: str = "",
95
        enable_auto_tools: bool = False,
96
        exclude_tools_when_tool_choice_none: bool = False,
97
        tool_parser: str | None = None,
98
        enable_prompt_tokens_details: bool = False,
99
        enable_force_include_usage: bool = False,
100
        enable_log_outputs: bool = False,
101
        log_error_stack: bool = False,
102
    ) -> None:
103
104
105
106
107
108
109
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
110

111
        self.response_role = response_role
112
113
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
114
        self.trust_request_chat_template = trust_request_chat_template
115
        self.enable_log_outputs = enable_log_outputs
116

117
118
119
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

120
121
122
123
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
124
125
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
126
127
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
128
129
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
130

131
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
132
        self.enable_force_include_usage = enable_force_include_usage
133
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
134
        if self.default_sampling_params:
135
136
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
137
138
139
140
141
142
143
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
144
        else:
145
            self.tool_call_id_type = "random"
146

147
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
148
149
150
151
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
152
153
                get_stop_tokens_for_assistant_actions()
            )
154
155
156
157
158
159
160
161
162
163
164

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
    async def warmup(self) -> None:
        """
        Warm up the chat template processing to avoid first-request latency.

        This method triggers Jinja2 template compilation and content format
        detection that would otherwise happen on the first real request,
        causing increased latency on the first request.
        """
        logger.info("Warming up chat template processing...")
        start_time = time.perf_counter()

        try:
            # Get the tokenizer from the engine
            tokenizer = await self.engine_client.get_tokenizer()

            # Create a minimal dummy request
            dummy_request = ChatCompletionRequest(
                messages=[{"role": "user", "content": "warmup"}],
                model=None,
                max_completion_tokens=1,
            )

            # Call _preprocess_chat to trigger template compilation
            # This forces:
            # 1. Chat template content format detection
            # 2. Jinja2 template compilation
            # 3. Tokenizer initialization for chat
            await self._preprocess_chat(
                dummy_request,
                tokenizer,
                dummy_request.messages,
                chat_template=self.chat_template,
                chat_template_content_format=self.chat_template_content_format,
                add_generation_prompt=True,
                continue_final_message=False,
                tool_dicts=None,
                documents=None,
                chat_template_kwargs=None,
                tool_parser=None,
                add_special_tokens=False,
            )

            elapsed = (time.perf_counter() - start_time) * 1000
            logger.info("Chat template warmup completed in %.1fms", elapsed)

        except Exception:
            # Log but don't fail server startup if warmup fails
            logger.exception("Chat template warmup failed")

214
    async def create_chat_completion(
215
216
        self,
        request: ChatCompletionRequest,
217
218
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
219
220
        """
        Chat Completion API similar to OpenAI's API.
221

222
223
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
224
        Chat Completion API.
225
226
227
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
228
            logger.error("Error with model %s", error_check_ret)
229
230
            return error_check_ret

231
232
233
234
235
236
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

237
        try:
238
            lora_request = self._maybe_get_adapters(
239
240
                request, supports_default_mm_loras=True
            )
241

242
            model_name = self.models.model_name(lora_request)
243

244
            tokenizer = await self.engine_client.get_tokenizer()
245

246
247
            tool_parser = self.tool_parser

248
            if isinstance(tokenizer, MistralTokenizer):
249
250
251
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
252
                maybe_serialize_tool_calls(request)
253
                truncate_tool_call_ids(request)
254
                validate_request_params(request)
255

256
257
258
            # Check if tool parsing is unavailable (common condition)
            tool_parsing_unavailable = (
                tool_parser is None
259
260
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
261
262
263
264
265
266
            )

            # Validate tool_choice when tool parsing is required but unavailable
            if tool_parsing_unavailable and request.tool_choice not in (
                None,
                "none",
267
            ):
268
269
270
271
272
273
274
275
276
277
278
279
280
                if request.tool_choice == "auto" and not self.enable_auto_tools:
                    # for hf tokenizers, "auto" tools requires
                    # --enable-auto-tool-choice and --tool-call-parser
                    return self.create_error_response(
                        '"auto" tool choice requires '
                        "--enable-auto-tool-choice and --tool-call-parser to be set"
                    )
                elif request.tool_choice != "auto":
                    # "required" or named tool requires tool parser
                    return self.create_error_response(
                        f'tool_choice="{request.tool_choice}" requires '
                        "--tool-call-parser to be set"
                    )
281

282
283
284
285
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
286
287
288
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
289

290
291
            if not self.use_harmony:
                # Common case.
292
293
294
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
295
                    trust_request_chat_template=self.trust_request_chat_template,
296
297
298
                )
                if error_check_ret is not None:
                    return error_check_ret
299
                conversation, engine_prompts = await self._preprocess_chat(
300
301
302
                    request,
                    tokenizer,
                    request.messages,
303
                    chat_template=request.chat_template or self.chat_template,
304
                    chat_template_content_format=self.chat_template_content_format,
305
306
307
308
309
310
311
312
313
314
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
315
316
317
318
                should_include_tools = tool_dicts is not None
                conversation, engine_prompts = self._make_request_with_harmony(
                    request, should_include_tools
                )
319
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
320
            logger.exception("Error in preprocessing prompt inputs")
321
            return self.create_error_response(f"{e} {e.__cause__}")
322

323
324
325
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
326
327
328
329
330

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

331
332
333
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

334
        # Schedule the request and get the result generator.
335
        generators: list[AsyncGenerator[RequestOutput, None]] = []
336
        try:
337
            for i, engine_prompt in enumerate(engine_prompts):
338
                prompt_text, _, _ = self._get_prompt_components(engine_prompt)
339
340
341
342
343
                # If we are creating sub requests for multiple prompts, ensure that they
                # have unique request ids.
                sub_request_id = (
                    request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                )
344
345
346
347
348
349
350
351

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
352
353
                    default_sampling_params=self.default_sampling_params,
                )
354

355
                sampling_params: SamplingParams | BeamSearchParams
356
357
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
358
359
                        max_tokens, self.default_sampling_params
                    )
360
361
                else:
                    sampling_params = request.to_sampling_params(
362
363
364
365
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
366
367
368
369
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
370

371
                self._log_inputs(
372
                    sub_request_id,
373
                    engine_prompt,
374
375
376
                    params=sampling_params,
                    lora_request=lora_request,
                )
377

378
379
380
381
382
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
383
384

                if isinstance(sampling_params, BeamSearchParams):
385
                    generator = self.beam_search(
386
                        prompt=engine_prompt,
387
                        request_id=sub_request_id,
388
                        params=sampling_params,
389
                        lora_request=lora_request,
390
                        trace_headers=trace_headers,
391
392
                    )
                else:
393
                    engine_request, tokenization_kwargs = await self._process_inputs(
394
                        sub_request_id,
395
396
397
398
399
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
400
                        data_parallel_rank=data_parallel_rank,
401
                    )
402

403
                    generator = self.engine_client.generate(
404
                        engine_request,
405
                        sampling_params,
406
                        sub_request_id,
407
408
409
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
410
411
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
412
                        data_parallel_rank=data_parallel_rank,
413
414
415
                    )

                generators.append(generator)
416
        except ValueError as e:
417
            # TODO: Use a vllm-specific Validation Error
418
419
            return self.create_error_response(str(e))

420
        assert len(generators) == 1
421
        (result_generator,) = generators
422

423
424
425
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
426
427
428
429
430
431
432
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
433
            )
434

435
436
        try:
            return await self.chat_completion_full_generator(
437
438
439
440
441
442
443
444
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
445
446
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
447
448
449
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
450
451
452
453

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
454
        return request.messages[-1]["role"]
455

456
    @staticmethod
457
    def _bracket_level(s: str, opening="{", closing="}") -> int:
458
459
460
461
462
463
464
465
466
467
468
469
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
470
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
471
472
473
474
475
476
477
478
479
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
480
            if c == "{":
481
482
                bracket_level += 1
                passed_zero = bracket_level == 0
483
            elif c == "}":
484
485
486
487
488
489
490
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
491
                if c == ",":
492
493
494
495
496
497
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
498
        current_text: str | None,
499
500
        delta_text: str,
        function_name_returned: bool,
501
502
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
503
504
505
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
506
507
508
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
509
            logger.debug("not enough tokens to parse into JSON yet")
510
511
512
513
514
515
516
517
518
519
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
520
521
                delta_text, previous_text
            )
522
523
524
525
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
526
527
528
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
529
530
531
532
533
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
534
535
536
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
537
538
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
539
540
                        arguments, previous_text
                    )
541
542
543
544

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
545
                    if finishes_previous_tool and "parameters" not in current_tool_call:
546
547
548
                        current_tool_call = obj[-2]

                    function_name_returned = True
549
550
551
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
552
553
554
555
556
557
558
559
560
561
562
563
564
565
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
566
567
568

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
569
570
                        delta_text, previous_text
                    )
571
572

                    if delta_text != "":
573
574
575
576
577
578
579
580
581
582
583
584
585
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
586
587
588
589
590
                    else:
                        delta_message = None

        return delta_message, function_name_returned

591
    async def chat_completion_stream_generator(
592
593
594
595
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
596
        model_name: str,
597
        conversation: list[ConversationMessage],
598
        tokenizer: TokenizerLike | None,
599
        request_metadata: RequestResponseMetadata,
600
    ) -> AsyncGenerator[str, None]:
601
        created_time = int(time.time())
602
        chunk_object_type: Final = "chat.completion.chunk"
603
        first_iteration = True
604
605

        # Send response for each token for each request.n (index)
606
607
608
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
609
        num_prompt_tokens = 0
610
        num_cached_tokens = None
611
612
        if self.use_harmony:
            harmony_parsers = [
613
                get_streamable_parser_for_assistant() for _ in range(num_choices)
614
            ]
615
616
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
617
618
619
620
621
622
623
624
625

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
626
627
            and self._should_stream_with_auto_tool_parsing(request)
        )
628

629
        all_previous_token_ids: list[list[int]] | None
630
        function_name_returned = [False] * num_choices
631
        if self.tool_call_id_type == "kimi_k2":
632
633
634
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
635

636
637
638
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

639
640
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
641
        if tool_choice_auto or self.reasoning_parser:
642
643
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
644
645
646
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
647
        else:
648
            all_previous_token_ids = None
649

650
        try:
651
            if self.reasoning_parser:
652
653
654
655
656
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

657
658
659
660
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
                    chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                )
661
662
663
664
665
666
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
667
668
669
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
670
671
672
673
674
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

675
                tool_parsers: list[ToolParser | None] = [
676
677
678
679
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
680
        except Exception as e:
681
            logger.exception("Error in tool parser creation.")
682
683
684
685
686
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

687
        stream_options = request.stream_options
688
689
690
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
691

692
693
        try:
            async for res in result_generator:
694
695
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
696
697
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
698

699
700
701
702
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
703
                    num_cached_tokens = res.num_cached_tokens
704
705
                    # Send first response for each request.n (index) with
                    # the role
706
                    role = self.get_chat_request_role(request)
707
708
709

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
710
                    for i in range(num_choices):
711
712
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
713
714
715
716
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
717
                            logprobs=None,
718
719
                            finish_reason=None,
                        )
720
721

                        # return prompt_token_ids at the first chunk ever
722
723
724
725
726
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
727
                            model=model_name,
728
729
730
731
732
733
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
734

735
736
737
738
739
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
740
741
                                total_tokens=num_prompt_tokens,
                            )
742

743
744
745
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

746
747
                    # Send response to echo the input portion of the
                    # last message
748
                    if request.echo:
749
                        last_msg_content: str | list[dict[str, str]] = ""
750
751
752
753
754
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
755
                            last_msg_content = conversation[-1]["content"] or ""
756
757

                        if last_msg_content:
758
                            for i in range(num_choices):
759
760
761
762
763
764
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
765
766
767
768
769
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
770
771
                                    model=model_name,
                                )
772
773
774
775
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
776
777
                                        total_tokens=num_prompt_tokens,
                                    )
778

779
                                data = chunk.model_dump_json(exclude_unset=True)
780
781
782
783
784
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
785
                    tool_parser = tool_parsers[i]
786
787
788
789

                    if finish_reason_sent[i]:
                        continue

790
                    if request.logprobs and request.top_logprobs is not None:
791
                        assert output.logprobs is not None, "Did not output logprobs"
792
                        logprobs = self._create_chat_logprobs(
793
794
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
795
                            tokenizer=tokenizer,
796
                            num_output_top_logprobs=request.top_logprobs,
797
                            return_as_token_id=request.return_tokens_as_token_ids,
798
799
800
801
                        )
                    else:
                        logprobs = None

802
803
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
804
                        prev_recipient = harmony_parser.current_recipient
805
                        delta_text = ""
806
807
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
808
                            delta_text += harmony_parser.last_content_delta or ""
809
810
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
811
812
                    else:
                        delta_text = output.text
813

814
815
816
817
818
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
819
820
821
                        # Chunked prefill case, don't return empty chunks
                        continue

822
                    delta_message: DeltaMessage | None
823

824
                    # just update previous_texts and previous_token_ids
825
                    if tool_choice_auto or self.reasoning_parser:
826
827
828
829
830
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
831
832
                        # avoid the None + list error.
                        if previous_token_ids:
833
                            current_token_ids = previous_token_ids + as_list(
834
835
                                output.token_ids
                            )
836
                        else:
837
                            current_token_ids = as_list(output.token_ids)
838

839
                    if self.use_harmony:
840
                        if cur_channel == "final":
841
                            delta_message = DeltaMessage(content=delta_text)
842
843
                        elif cur_channel == "analysis":
                            if request.include_reasoning:
844
                                delta_message = DeltaMessage(reasoning=delta_text)
845
846
                            else:
                                delta_message = None
847
848
849
850
851
                        elif (
                            cur_channel == "commentary"
                            and cur_recipient
                            and cur_recipient.startswith("functions.")
                        ):
852
853
854
                            # Count completed tool calls to determine index
                            base_index = 0
                            for msg in harmony_parser.messages:
855
856
857
858
859
                                if (
                                    msg.channel == "commentary"
                                    and msg.recipient
                                    and msg.recipient.startswith("functions.")
                                ):
860
861
862
                                    base_index += 1

                            if prev_recipient != cur_recipient:
863
864
865
866
867
868
869
870
871
872
873
874
875
876
                                tool_name = cur_recipient.split("functions.", 1)[1]
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            id=make_tool_call_id(),
                                            type="function",
                                            function=DeltaFunctionCall(
                                                name=tool_name,
                                                arguments="",
                                            ),
                                            index=base_index,
                                        )
                                    ]
                                )
877
                            elif delta_text:
878
879
880
881
882
883
884
885
886
887
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            index=base_index,
                                            function=DeltaFunctionCall(
                                                arguments=delta_text
                                            ),
                                        )
                                    ]
                                )
888
889
890
891
892
                            else:
                                delta_message = None

                            if delta_message is not None:
                                harmony_tools_streamed[i] = True
893
894
895
                        elif cur_channel == "commentary":
                            # Tool call preambles meant to be shown to the user
                            delta_message = DeltaMessage(content=delta_text)
896
897
                        else:
                            delta_message = None
898
                    # handle streaming deltas for tools with named tool_choice
899
                    elif tool_choice_function_name:
900
901
902
903
904
905
906
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
907
908
                            assert reasoning_parser is not None
                            delta_message = (
909
                                reasoning_parser.extract_reasoning_streaming(
910
911
912
913
914
915
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
916
917
                                )
                            )
918
919
920
921
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
922
                            # Only keep 'content', remove 'reasoning'.
923
                            if reasoning_parser.is_reasoning_end(
924
925
926
927
928
929
930
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
931
                                reasoning_end_arr[i] = True
932
933
934
935
936
937
938
939
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
940
                            if self.reasoning_parser:
941
942
943
                                delta_text = previous_text + delta_text
                                current_text = ""

944
945
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
946
947
948
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
949
950
                            else:
                                delta_tool_call = DeltaToolCall(
951
                                    id=make_tool_call_id(),
952
953
954
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
955
956
957
958
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
959
960
                                function_name_returned[i] = True

961
962
963
964
965
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
966
                            tools_streamed[i] = True
967

968
969
970
971
972
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
973
974
975
976
977
978
979
980
981
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
                            and res.prompt_token_ids
                            and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        ):
                            reasoning_end_arr[i] = True
982

983
984
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
985
                                reasoning_parser.extract_reasoning_streaming(
986
987
988
989
990
991
992
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
993
                            )
994
995
996
997
998
999
1000
1001
1002
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

1003
                        else:
1004
                            # either finished reasoning or no reasoning at all
1005
                            content = current_text
1006
1007
1008
1009
1010
1011
1012
1013
1014

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
1015
                            )
1016
1017
1018
1019
1020
1021
1022
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
1023

1024
1025
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
1026
                    elif tool_choice_auto and self.reasoning_parser:
1027
1028
1029
1030
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
1031
                        output_token_ids = as_list(output.token_ids)
1032
                        if not reasoning_end_arr[i]:
1033
1034
1035
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
1036
1037
1038
1039
1040
1041
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
1042
                                reasoning_end_arr[i] = True
1043
                                current_token_ids = output_token_ids
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
1054
1055
                                    )
                                )
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
1073
1074

                        # handle tool calls only after reasoning is done,
1075
                        if reasoning_end_arr[i]:
1076
                            delta_token_ids = output_token_ids
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1087
                            delta_message = tool_parser.extract_tool_calls_streaming(
1088
1089
                                previous_text=previous_text,
                                current_text=current_text,
1090
                                delta_text=delta_text,
1091
1092
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1110
1111
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1112

1113
                    # when only reasoning
1114
                    elif self.reasoning_parser:
1115
1116
1117
1118
1119
1120
1121
                        delta_message = reasoning_parser.extract_reasoning_streaming(
                            previous_text,
                            current_text,
                            delta_text,
                            previous_token_ids,
                            current_token_ids,
                            output.token_ids,
1122
                        )
1123
                    # handle streaming just a content delta
1124
1125
1126
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1127
                    # update the previous values for the next iteration
1128
1129
1130
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1131
1132
1133
1134
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1135
1136
1137
1138
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1139

1140
                    # set the previous values for the next iteration
1141
                    previous_num_tokens[i] += len(output.token_ids)
1142
1143
1144
1145
1146
1147

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1148
1149
1150
1151
1152
1153
1154
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1155
                            continue
1156
                        delta_message = DeltaMessage()
1157

1158
1159
1160
1161
1162
1163
1164
1165
1166
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1167
1168
                                if tc.function and tc.function.arguments
                            )
1169
1170
1171
1172
1173

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1174
                                output_token_ids=as_list(output.token_ids),
1175
1176
1177
1178
1179
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1180
1181
1182
1183
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1184
                            delta=delta_message,
1185
                            logprobs=logprobs,
1186
                            finish_reason=None,
1187
1188
1189
1190
1191
1192
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1193
1194

                    # if the model is finished generating
1195
                    else:
1196
1197
1198
1199
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1200
1201
1202
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1203
                        # only happens if we are NOT using structured outputs
1204
                        auto_tools_called = False
1205
                        if tool_parser:
1206
1207
1208
1209
1210
1211
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1212
1213
1214
                        else:
                            index = 0

1215
1216
1217
1218
1219
1220
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1221
                            latest_delta_len = 0
1222
1223
                            if (
                                isinstance(
1224
                                    delta_message.tool_calls[0].function,
1225
1226
1227
1228
1229
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1230
                                latest_delta_len = len(
1231
1232
                                    delta_message.tool_calls[0].function.arguments
                                )
1233

1234
1235
1236
1237
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1238
1239
1240
1241
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1242

1243
                            # get what we've streamed so far for arguments
1244
                            # for the current tool
1245
1246
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1247
                                actual_call = actual_call[:-latest_delta_len]
1248
1249

                            # check to see if there's anything left to stream
1250
                            remaining_call = expected_call.replace(actual_call, "", 1)
1251
                            # set that as a delta message
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    DeltaToolCall(
                                        index=index,
                                        function=DeltaFunctionCall(
                                            arguments=remaining_call
                                        ).model_dump(exclude_none=True),
                                    )
                                ]
                            )
1262

1263
                        # Send the finish response for each request.n only once
1264
1265
1266
1267
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1268
1269
                        if (
                            auto_tools_called
1270
                            or (tools_streamed[i] and not tool_choice_function_name)
1271
1272
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1273
1274
                            finish_reason_ = "tool_calls"
                        else:
1275
1276
1277
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1278
1279
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1280
                            delta=delta_message,
1281
                            logprobs=logprobs,
1282
                            finish_reason=finish_reason_,
1283
                            stop_reason=output.stop_reason,
1284
1285
1286
1287
1288
1289
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1290

1291
                        finish_reason_sent[i] = True
1292

1293
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1294
1295
1296
1297
1298
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1299
1300
                        model=model_name,
                    )
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1311
                    data = chunk.model_dump_json(exclude_unset=True)
1312
1313
                    yield f"data: {data}\n\n"

1314
1315
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1316
1317
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1318
1319
1320
1321
1322
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1323
1324
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1325
1326
                        cached_tokens=num_cached_tokens
                    )
1327
1328
1329
1330
1331
1332
1333

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1334
1335
1336
1337
1338
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1339
                yield f"data: {final_usage_data}\n\n"
1340

1341
1342
1343
1344
1345
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1346
1347
1348
1349
1350
1351
1352
1353
1354
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1355
1356
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1357
1358
1359
1360
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1361
                        output_token_ids=None,  # Consider also logging all token IDs
1362
1363
1364
1365
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1366

1367
1368
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1369
        except Exception as e:
1370
            # TODO: Use a vllm-specific Validation Error
1371
            logger.exception("Error in chat completion stream generator.")
1372
1373
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1374
1375
1376
1377
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1378
1379
1380
1381
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1382
        model_name: str,
1383
        conversation: list[ConversationMessage],
1384
        tokenizer: TokenizerLike | None,
1385
        request_metadata: RequestResponseMetadata,
1386
    ) -> ErrorResponse | ChatCompletionResponse:
1387
        created_time = int(time.time())
1388
        final_res: RequestOutput | None = None
1389

1390
1391
1392
1393
1394
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1395
1396
1397
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1398

1399
1400
        assert final_res is not None

1401
        choices: list[ChatCompletionResponseChoice] = []
1402
        if self.tool_call_id_type == "kimi_k2":
1403
1404
1405
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1406

1407
1408
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1409
1410
1411
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1412
            token_ids = output.token_ids
1413
            out_logprobs = output.logprobs
1414
            tool_call_info = None
1415

1416
1417
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1418
                logprobs = self._create_chat_logprobs(
1419
                    token_ids=token_ids,
1420
                    top_logprobs=out_logprobs,
1421
                    num_output_top_logprobs=request.top_logprobs,
1422
                    tokenizer=tokenizer,
1423
                    return_as_token_id=request.return_tokens_as_token_ids,
1424
1425
1426
                )
            else:
                logprobs = None
1427
1428

            if self.use_harmony:
1429
                reasoning, content, _ = parse_chat_output(token_ids)
1430
                if not request.include_reasoning:
1431
                    reasoning = None
1432

1433
                if self.tool_parser is not None:
1434
1435
1436
1437
1438
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1439
1440
1441
1442
1443
1444
1445
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1446
                    content = tool_call_info.content
1447
1448
                    message = ChatMessage(
                        role=role,
1449
                        reasoning=reasoning,
1450
1451
1452
1453
1454
1455
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1456
                        reasoning=reasoning,
1457
1458
                        content=content,
                    )
1459
1460
1461
1462
1463

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1464
1465
1466
1467
1468
1469
1470
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1471
                    stop_reason=output.stop_reason,
1472
1473
1474
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1475
1476
1477
                )
                choices.append(choice_data)
                continue
1478

1479
            if self.reasoning_parser:
1480
                try:
1481
1482
1483
1484
1485
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1486
1487
1488
1489
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
                        chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                    )
1490
1491
1492
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1493
1494
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1495
                reasoning, content = reasoning_parser.extract_reasoning(
1496
1497
                    output.text, request=request
                )
1498
                if not request.include_reasoning:
1499
                    reasoning = None
1500
            else:
1501
                reasoning = None
1502
                content = output.text
1503

1504
            auto_tools_called = False
1505
1506
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1517
1518
1519
1520
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1521
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1522
1523

            # if the request uses tools and specified a tool choice
1524
1525
1526
1527
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1528
                assert tool_calls is not None and len(tool_calls) > 0
1529
1530
                message = ChatMessage(
                    role=role,
1531
                    reasoning=reasoning,
1532
                    content="",
1533
                    tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1534
                )
1535

1536
            elif request.tool_choice and request.tool_choice == "required":
1537
1538
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1539
                for tool_call in tool_calls:
1540
1541
1542
1543
1544
1545
1546
1547
                    tool_call_class_items.append(
                        tool_call_class(
                            id=make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
                                idx=history_tool_call_cnt,
                            ),
                            function=tool_call,
1548
1549
                        )
                    )
1550
                    history_tool_call_cnt += 1
1551
1552
1553
                message = ChatMessage(
                    role=role,
                    content="",
1554
                    tool_calls=tool_call_class_items,
1555
                    reasoning=reasoning,
1556
                )
1557

1558
1559
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1560
            elif not request.tool_choice or request.tool_choice == "none":
1561
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1562
1563

            # handle when there are tools and tool choice is auto
1564
1565
1566
1567
1568
1569
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1570
1571
1572
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1573
1574
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1575
1576
                    message = ChatMessage(
                        role=role,
1577
                        reasoning=reasoning,
1578
1579
1580
1581
1582
1583
1584
1585
                        content=content,
                        tool_calls=[
                            ToolCall(
                                function=tc,
                                type="function",
                            )
                            for tc in tool_calls
                        ],
1586
                    )
1587
1588
1589
1590

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1591
1592
1593
1594
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1595
1596
                    if content and len(content) > 0:
                        ret_content = content
1597
1598
                    message = ChatMessage(
                        role=role,
1599
                        reasoning=reasoning,
1600
1601
                        content=ret_content,
                    )
1602
1603
1604
1605
1606
1607

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1608
1609
                    "completion."
                )
1610
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1611
1612
1613
1614
1615
1616
1617
1618
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1619

1620
1621
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1622
                message=message,
1623
                logprobs=logprobs,
1624
1625
1626
1627
1628
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1629
                stop_reason=output.stop_reason,
1630
1631
1632
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1633
            )
1634
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1635

1636
1637
            choices.append(choice_data)

1638
        if request.echo:
1639
            last_msg_content: str | list[dict[str, str]] = ""
1640
1641
1642
1643
1644
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1645
                last_msg_content = conversation[-1]["content"] or ""
1646
            if isinstance(last_msg_content, list):
1647
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1648
1649

            for choice in choices:
1650
                full_message = last_msg_content + (choice.message.content or "")
1651
1652
                choice.message.content = full_message

1653
        assert final_res.prompt_token_ids is not None
1654
        num_prompt_tokens = len(final_res.prompt_token_ids)
1655
1656
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1657
        num_generated_tokens = sum(
1658
1659
1660
1661
1662
1663
1664
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1665
1666
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1667
1668
                cached_tokens=final_res.num_cached_tokens
            )
1669
1670
1671

        request_metadata.final_usage_info = usage

1672
1673
1674
1675
1676
1677
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1678
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1679
1680
1681
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1682
            kv_transfer_params=final_res.kv_transfer_params,
1683
1684
        )

1685
1686
1687
1688
1689
1690
1691
1692
1693
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1694
1695
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1696
1697
                            tc.function, "arguments"
                        ):
1698
                            tool_call_descriptions.append(
1699
1700
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1701
1702
1703
1704
1705
1706
1707
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1708
                        output_token_ids = final_res.outputs[choice.index].token_ids
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1719
        return response
1720
1721

    def _get_top_logprobs(
1722
1723
        self,
        logprobs: dict[int, Logprob],
1724
        top_logprobs: int | None,
1725
        tokenizer: TokenizerLike | None,
1726
1727
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1728
        return [
1729
            ChatCompletionLogProb(
1730
1731
1732
1733
1734
1735
1736
1737
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1738
1739
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1740
1741
            )
            for i, p in enumerate(logprobs.items())
1742
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1743
1744
1745
1746
1747
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1748
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1749
        tokenizer: TokenizerLike | None,
1750
1751
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1752
1753
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1754
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1755

1756
1757
1758
1759
1760
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1761
1762
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1763
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1764
                if should_return_as_token_id:
1765
                    token = f"token_id:{token_id}"
1766
                else:
1767
1768
1769
1770
1771
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1772
                    token = tokenizer.decode(token_id)
1773

1774
1775
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1776
                        token=token,
1777
                        bytes=list(token.encode("utf-8", errors="replace")),
1778
1779
                    )
                )
1780
            else:
1781
1782
1783
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1784
1785
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1786
                        token=self._get_decoded_token(
1787
1788
1789
                            step_token,
                            token_id,
                            tokenizer,
1790
                            should_return_as_token_id,
1791
1792
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1793
1794
1795
1796
1797
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1798
                        top_logprobs=self._get_top_logprobs(
1799
1800
1801
1802
1803
1804
1805
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1806
1807

        return ChatCompletionLogProbs(content=logprobs_content)
1808

1809
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1810
1811
1812
1813
1814
1815
1816
1817
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1818
1819
1820
1821
1822
1823
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1824
1825
1826

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1827
        delta_message: DeltaMessage | None,
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1839
            output.finish_reason is not None
1840
1841
1842
1843
1844
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1845
1846
1847
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1848
1849
1850
1851

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
1852
        should_include_tools: bool = True,
1853
1854
1855
    ):
        messages: list[OpenAIMessage] = []

1856
1857
1858
1859
1860
        # because of issues with pydantic we need to potentially
        # re-serialize the tool_calls field of the request
        # for more info: see comment in `maybe_serialize_tool_calls`
        maybe_serialize_tool_calls(request)

1861
1862
1863
1864
1865
1866
1867
1868
        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1869
            python_description=None,
1870
            with_custom_tools=should_include_tools,
1871
        )
1872
1873
1874
        messages.append(sys_msg)

        # Add developer message.
1875
1876
1877
        dev_msg = get_developer_message(
            tools=request.tools if should_include_tools else None
        )
1878
1879
1880
        messages.append(dev_msg)

        # Add user message.
1881
        messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1882
1883
1884

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
1885
        engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1886
1887
1888
1889
1890

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1891
        return messages, [engine_prompt]