serving_chat.py 80.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
22
23
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
24
25
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.parser.harmony_utils import (
26
27
28
29
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
30
    parse_chat_inputs_to_harmony_messages,
31
32
33
    parse_chat_output,
    render_for_completion,
)
34
from vllm.entrypoints.openai.protocol import (
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
54
55
56
from vllm.entrypoints.openai.serving_chat_stream_harmony import (
    extract_harmony_streaming_delta,
)
57
58
59
60
61
from vllm.entrypoints.openai.serving_engine import (
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
62
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
63
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
64
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
65
from vllm.inputs.data import TokensPrompt
66
from vllm.logger import init_logger
67
from vllm.logprobs import Logprob
68
from vllm.outputs import CompletionOutput, RequestOutput
69
from vllm.sampling_params import BeamSearchParams, SamplingParams
70
71
72
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
    MistralTokenizer,
73
74
75
76
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
77
78
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
79
from vllm.utils.collection_utils import as_list
80
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
81
82
83
84
85

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
86
87
88
    def __init__(
        self,
        engine_client: EngineClient,
89
        models: OpenAIServingModels,
90
91
        response_role: str,
        *,
92
93
        request_logger: RequestLogger | None,
        chat_template: str | None,
94
        chat_template_content_format: ChatTemplateContentFormatOption,
95
        trust_request_chat_template: bool = False,
96
        return_tokens_as_token_ids: bool = False,
97
        reasoning_parser: str = "",
98
        enable_auto_tools: bool = False,
99
        exclude_tools_when_tool_choice_none: bool = False,
100
        tool_parser: str | None = None,
101
        enable_prompt_tokens_details: bool = False,
102
        enable_force_include_usage: bool = False,
103
        enable_log_outputs: bool = False,
104
        log_error_stack: bool = False,
105
    ) -> None:
106
107
108
109
110
111
112
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
113

114
        self.response_role = response_role
115
116
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
117
        self.trust_request_chat_template = trust_request_chat_template
118
        self.enable_log_outputs = enable_log_outputs
119

120
121
122
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

123
124
125
126
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
127
128
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
129
130
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
131
132
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
133

134
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
135
        self.enable_force_include_usage = enable_force_include_usage
136
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
137
        if self.default_sampling_params:
138
139
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
140
141
142
143
144
145
146
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
147
        else:
148
            self.tool_call_id_type = "random"
149

150
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
151
152
153
154
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
155
156
                get_stop_tokens_for_assistant_actions()
            )
157
158
159
160
161
162
163
164
165
166
167

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
    async def warmup(self) -> None:
        """
        Warm up the chat template processing to avoid first-request latency.

        This method triggers Jinja2 template compilation and content format
        detection that would otherwise happen on the first real request,
        causing increased latency on the first request.
        """
        logger.info("Warming up chat template processing...")
        start_time = time.perf_counter()

        try:
            # Get the tokenizer from the engine
            tokenizer = await self.engine_client.get_tokenizer()

            # Create a minimal dummy request
            dummy_request = ChatCompletionRequest(
                messages=[{"role": "user", "content": "warmup"}],
                model=None,
                max_completion_tokens=1,
            )

            # Call _preprocess_chat to trigger template compilation
            # This forces:
            # 1. Chat template content format detection
            # 2. Jinja2 template compilation
            # 3. Tokenizer initialization for chat
            await self._preprocess_chat(
                dummy_request,
                tokenizer,
                dummy_request.messages,
                chat_template=self.chat_template,
                chat_template_content_format=self.chat_template_content_format,
                add_generation_prompt=True,
                continue_final_message=False,
                tool_dicts=None,
                documents=None,
                chat_template_kwargs=None,
                tool_parser=None,
                add_special_tokens=False,
            )

            elapsed = (time.perf_counter() - start_time) * 1000
            logger.info("Chat template warmup completed in %.1fms", elapsed)

        except Exception:
            # Log but don't fail server startup if warmup fails
            logger.exception("Chat template warmup failed")

217
    async def create_chat_completion(
218
219
        self,
        request: ChatCompletionRequest,
220
221
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
222
223
        """
        Chat Completion API similar to OpenAI's API.
224

225
226
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
227
        Chat Completion API.
228
229
230
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
231
            logger.error("Error with model %s", error_check_ret)
232
233
            return error_check_ret

234
235
236
237
238
239
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

240
        try:
241
            lora_request = self._maybe_get_adapters(
242
243
                request, supports_default_mm_loras=True
            )
244

245
            model_name = self.models.model_name(lora_request)
246

247
            tokenizer = await self.engine_client.get_tokenizer()
248

249
250
            tool_parser = self.tool_parser

251
            if isinstance(tokenizer, MistralTokenizer):
252
253
254
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
255
                maybe_serialize_tool_calls(request)
256
                truncate_tool_call_ids(request)
257
                validate_request_params(request)
258

259
260
261
            # Check if tool parsing is unavailable (common condition)
            tool_parsing_unavailable = (
                tool_parser is None
262
263
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
264
265
266
267
268
269
            )

            # Validate tool_choice when tool parsing is required but unavailable
            if tool_parsing_unavailable and request.tool_choice not in (
                None,
                "none",
270
            ):
271
272
273
274
275
276
277
278
279
280
281
282
283
                if request.tool_choice == "auto" and not self.enable_auto_tools:
                    # for hf tokenizers, "auto" tools requires
                    # --enable-auto-tool-choice and --tool-call-parser
                    return self.create_error_response(
                        '"auto" tool choice requires '
                        "--enable-auto-tool-choice and --tool-call-parser to be set"
                    )
                elif request.tool_choice != "auto":
                    # "required" or named tool requires tool parser
                    return self.create_error_response(
                        f'tool_choice="{request.tool_choice}" requires '
                        "--tool-call-parser to be set"
                    )
284

285
286
287
288
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
289
290
291
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
292

293
294
            if not self.use_harmony:
                # Common case.
295
296
297
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
298
                    trust_request_chat_template=self.trust_request_chat_template,
299
300
301
                )
                if error_check_ret is not None:
                    return error_check_ret
302
                conversation, engine_prompts = await self._preprocess_chat(
303
304
305
                    request,
                    tokenizer,
                    request.messages,
306
                    chat_template=request.chat_template or self.chat_template,
307
                    chat_template_content_format=self.chat_template_content_format,
308
309
310
311
312
313
314
315
316
317
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
318
319
320
321
                should_include_tools = tool_dicts is not None
                conversation, engine_prompts = self._make_request_with_harmony(
                    request, should_include_tools
                )
322
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
323
            logger.exception("Error in preprocessing prompt inputs")
324
            return self.create_error_response(f"{e} {e.__cause__}")
325

326
327
328
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
329
330
331
332
333

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

334
335
336
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

337
        # Schedule the request and get the result generator.
338
        generators: list[AsyncGenerator[RequestOutput, None]] = []
339
        try:
340
            for i, engine_prompt in enumerate(engine_prompts):
341
                prompt_text, _, _ = self._get_prompt_components(engine_prompt)
342
343
344
345
346
                # If we are creating sub requests for multiple prompts, ensure that they
                # have unique request ids.
                sub_request_id = (
                    request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                )
347
348
349
350
351
352
353
354

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
355
356
                    default_sampling_params=self.default_sampling_params,
                )
357

358
                sampling_params: SamplingParams | BeamSearchParams
359
360
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
361
362
                        max_tokens, self.default_sampling_params
                    )
363
364
                else:
                    sampling_params = request.to_sampling_params(
365
366
367
368
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
369
370
371
372
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
373

374
                self._log_inputs(
375
                    sub_request_id,
376
                    engine_prompt,
377
378
379
                    params=sampling_params,
                    lora_request=lora_request,
                )
380

381
382
383
384
385
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
386
387

                if isinstance(sampling_params, BeamSearchParams):
388
                    generator = self.beam_search(
389
                        prompt=engine_prompt,
390
                        request_id=sub_request_id,
391
                        params=sampling_params,
392
                        lora_request=lora_request,
393
                        trace_headers=trace_headers,
394
395
                    )
                else:
396
                    engine_request, tokenization_kwargs = await self._process_inputs(
397
                        sub_request_id,
398
399
400
401
402
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
403
                        data_parallel_rank=data_parallel_rank,
404
                    )
405

406
                    generator = self.engine_client.generate(
407
                        engine_request,
408
                        sampling_params,
409
                        sub_request_id,
410
411
412
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
413
414
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
415
                        data_parallel_rank=data_parallel_rank,
416
417
418
                    )

                generators.append(generator)
419
        except ValueError as e:
420
            return self.create_error_response(e)
421

422
        assert len(generators) == 1
423
        (result_generator,) = generators
424

425
426
427
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
428
429
430
431
432
433
434
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
435
            )
436

437
438
        try:
            return await self.chat_completion_full_generator(
439
440
441
442
443
444
445
446
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
447
448
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
449
        except ValueError as e:
450
            return self.create_error_response(e)
451
452
453
454

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
455
        return request.messages[-1]["role"]
456

457
    @staticmethod
458
    def _bracket_level(s: str, opening="{", closing="}") -> int:
459
460
461
462
463
464
465
466
467
468
469
470
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
471
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
472
473
474
475
476
477
478
479
480
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
481
            if c == "{":
482
483
                bracket_level += 1
                passed_zero = bracket_level == 0
484
            elif c == "}":
485
486
487
488
489
490
491
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
492
                if c == ",":
493
494
495
496
497
498
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
499
        current_text: str | None,
500
501
        delta_text: str,
        function_name_returned: bool,
502
503
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
504
505
506
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
507
508
509
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
510
            logger.debug("not enough tokens to parse into JSON yet")
511
512
513
514
515
516
517
518
519
520
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
521
522
                delta_text, previous_text
            )
523
524
525
526
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
527
528
529
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
530
531
532
533
534
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
535
536
537
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
538
539
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
540
541
                        arguments, previous_text
                    )
542
543
544
545

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
546
                    if finishes_previous_tool and "parameters" not in current_tool_call:
547
548
549
                        current_tool_call = obj[-2]

                    function_name_returned = True
550
551
552
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
553
554
555
556
557
558
559
560
561
562
563
564
565
566
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
567
568
569

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
570
571
                        delta_text, previous_text
                    )
572
573

                    if delta_text != "":
574
575
576
577
578
579
580
581
582
583
584
585
586
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
587
588
589
590
591
                    else:
                        delta_message = None

        return delta_message, function_name_returned

592
    async def chat_completion_stream_generator(
593
594
595
596
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
597
        model_name: str,
598
        conversation: list[ConversationMessage],
599
        tokenizer: TokenizerLike | None,
600
        request_metadata: RequestResponseMetadata,
601
    ) -> AsyncGenerator[str, None]:
602
        created_time = int(time.time())
603
        chunk_object_type: Final = "chat.completion.chunk"
604
        first_iteration = True
605
606

        # Send response for each token for each request.n (index)
607
608
609
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
610
        num_prompt_tokens = 0
611
        num_cached_tokens = None
612
613
        if self.use_harmony:
            harmony_parsers = [
614
                get_streamable_parser_for_assistant() for _ in range(num_choices)
615
            ]
616
617
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
618
619
620
621
622
623
624
625
626

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
627
628
            and self._should_stream_with_auto_tool_parsing(request)
        )
629

630
        all_previous_token_ids: list[list[int]] | None
631
        function_name_returned = [False] * num_choices
632
        if self.tool_call_id_type == "kimi_k2":
633
634
635
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
636

637
638
639
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

640
641
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
642
        if tool_choice_auto or self.reasoning_parser:
643
644
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
645
646
647
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
648
        else:
649
            all_previous_token_ids = None
650

651
        try:
652
            if self.reasoning_parser:
653
654
655
656
657
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

658
659
660
661
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
                    chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                )
662
663
664
665
666
667
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
668
669
670
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
671
672
673
674
675
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

676
                tool_parsers: list[ToolParser | None] = [
677
678
679
680
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
681
        except Exception as e:
682
            logger.exception("Error in tool parser creation.")
683
            data = self.create_streaming_error_response(e)
684
685
686
687
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

688
        stream_options = request.stream_options
689
690
691
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
692

693
694
        try:
            async for res in result_generator:
695
696
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
697
698
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
699

700
701
702
703
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
704
                    num_cached_tokens = res.num_cached_tokens
705
706
                    # Send first response for each request.n (index) with
                    # the role
707
                    role = self.get_chat_request_role(request)
708
709
710

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
711
                    for i in range(num_choices):
712
713
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
714
715
716
717
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
718
                            logprobs=None,
719
720
                            finish_reason=None,
                        )
721
722

                        # return prompt_token_ids at the first chunk ever
723
724
725
726
727
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
728
                            model=model_name,
729
730
731
732
733
734
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
735

736
737
738
739
740
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
741
742
                                total_tokens=num_prompt_tokens,
                            )
743

744
745
746
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

747
748
                    # Send response to echo the input portion of the
                    # last message
749
                    if request.echo:
750
                        last_msg_content: str | list[dict[str, str]] = ""
751
752
753
754
755
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
756
                            last_msg_content = conversation[-1]["content"] or ""
757
758

                        if last_msg_content:
759
                            for i in range(num_choices):
760
761
762
763
764
765
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
766
767
768
769
770
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
771
772
                                    model=model_name,
                                )
773
774
775
776
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
777
778
                                        total_tokens=num_prompt_tokens,
                                    )
779

780
                                data = chunk.model_dump_json(exclude_unset=True)
781
782
783
784
785
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
786
                    tool_parser = tool_parsers[i]
787
788
789
790

                    if finish_reason_sent[i]:
                        continue

791
                    if request.logprobs and request.top_logprobs is not None:
792
                        assert output.logprobs is not None, "Did not output logprobs"
793
                        logprobs = self._create_chat_logprobs(
794
795
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
796
                            tokenizer=tokenizer,
797
                            num_output_top_logprobs=request.top_logprobs,
798
                            return_as_token_id=request.return_tokens_as_token_ids,
799
800
801
802
                        )
                    else:
                        logprobs = None

803
804
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
805
                        prev_recipient = harmony_parser.current_recipient
806
                        delta_text = ""
807
808
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
809
                            delta_text += harmony_parser.last_content_delta or ""
810
811
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
812
813
814
815
816
                        # handle the case where several tokens where generated at once
                        # including the final token, leading to a delta in the text
                        # but the current channel to be empty (start state)
                        if not cur_channel and delta_text:
                            cur_channel = "final"
817
818
                    else:
                        delta_text = output.text
819

820
821
822
823
824
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
825
826
827
                        # Chunked prefill case, don't return empty chunks
                        continue

828
                    delta_message: DeltaMessage | None
829

830
                    # just update previous_texts and previous_token_ids
831
                    if tool_choice_auto or self.reasoning_parser:
832
833
834
835
836
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
837
838
                        # avoid the None + list error.
                        if previous_token_ids:
839
                            current_token_ids = previous_token_ids + as_list(
840
841
                                output.token_ids
                            )
842
                        else:
843
                            current_token_ids = as_list(output.token_ids)
844

845
                    if self.use_harmony:
846
847
848
849
850
851
852
853
854
855
856
                        delta_message, tools_streamed_flag = (
                            extract_harmony_streaming_delta(
                                harmony_parser=harmony_parser,
                                cur_channel=cur_channel,
                                cur_recipient=cur_recipient,
                                prev_recipient=prev_recipient,
                                delta_text=delta_text,
                                include_reasoning=request.include_reasoning,
                            )
                        )
                        harmony_tools_streamed[i] |= tools_streamed_flag
857
                    # handle streaming deltas for tools with named tool_choice
858
                    elif tool_choice_function_name:
859
860
861
862
863
864
865
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
866
867
                            assert reasoning_parser is not None
                            delta_message = (
868
                                reasoning_parser.extract_reasoning_streaming(
869
870
871
872
873
874
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
875
876
                                )
                            )
877
878
879
880
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
881
                            # Only keep 'content', remove 'reasoning'.
882
                            if reasoning_parser.is_reasoning_end(
883
884
885
886
887
888
889
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
890
                                reasoning_end_arr[i] = True
891
892
893
894
895
896
897
898
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
899
                            if self.reasoning_parser:
900
901
902
                                delta_text = previous_text + delta_text
                                current_text = ""

903
904
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
905
906
907
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
908
909
                            else:
                                delta_tool_call = DeltaToolCall(
910
                                    id=make_tool_call_id(),
911
912
913
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
914
915
916
917
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
918
919
                                function_name_returned[i] = True

920
921
922
923
924
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
925
                            tools_streamed[i] = True
926

927
928
929
930
931
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
932
933
934
935
936
937
938
939
940
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
                            and res.prompt_token_ids
                            and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        ):
                            reasoning_end_arr[i] = True
941

942
943
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
944
                                reasoning_parser.extract_reasoning_streaming(
945
946
947
948
949
950
951
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
952
                            )
953
954
955
956
957
958
959
960
961
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

962
                        else:
963
                            # either finished reasoning or no reasoning at all
964
                            content = current_text
965
966
967
968
969
970
971
972
973

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
974
                            )
975
976
977
978
979
980
981
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
982

983
984
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
985
                    elif tool_choice_auto and self.reasoning_parser:
986
987
988
989
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
990
                        output_token_ids = as_list(output.token_ids)
991
                        if not reasoning_end_arr[i]:
992
993
994
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
995
996
997
998
999
1000
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
1001
                                reasoning_end_arr[i] = True
1002
                                current_token_ids = output_token_ids
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
1013
1014
                                    )
                                )
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
1032
1033

                        # handle tool calls only after reasoning is done,
1034
                        if reasoning_end_arr[i]:
1035
                            delta_token_ids = output_token_ids
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1046
                            delta_message = tool_parser.extract_tool_calls_streaming(
1047
1048
                                previous_text=previous_text,
                                current_text=current_text,
1049
                                delta_text=delta_text,
1050
1051
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1069
1070
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1071

1072
                    # when only reasoning
1073
                    elif self.reasoning_parser:
1074
1075
1076
1077
1078
1079
1080
                        delta_message = reasoning_parser.extract_reasoning_streaming(
                            previous_text,
                            current_text,
                            delta_text,
                            previous_token_ids,
                            current_token_ids,
                            output.token_ids,
1081
                        )
1082
                    # handle streaming just a content delta
1083
1084
1085
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1086
                    # update the previous values for the next iteration
1087
1088
1089
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1090
1091
1092
1093
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1094
1095
1096
1097
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1098

1099
                    # set the previous values for the next iteration
1100
                    previous_num_tokens[i] += len(output.token_ids)
1101
1102
1103
1104
1105
1106

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1107
1108
1109
1110
1111
1112
1113
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1114
                            continue
1115
                        delta_message = DeltaMessage()
1116

1117
1118
1119
1120
1121
1122
1123
1124
1125
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1126
1127
                                if tc.function and tc.function.arguments
                            )
1128
1129
1130
1131
1132

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1133
                                output_token_ids=as_list(output.token_ids),
1134
1135
1136
1137
1138
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1139
1140
1141
1142
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1143
                            delta=delta_message,
1144
                            logprobs=logprobs,
1145
                            finish_reason=None,
1146
1147
1148
1149
1150
1151
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1152
1153

                    # if the model is finished generating
1154
                    else:
1155
1156
1157
1158
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1159
1160
1161
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1162
                        # only happens if we are NOT using structured outputs
1163
                        auto_tools_called = False
1164
                        if tool_parser:
1165
1166
1167
1168
1169
1170
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1171
1172
1173
                        else:
                            index = 0

1174
1175
1176
1177
1178
1179
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1180
                            latest_delta_len = 0
1181
1182
                            if (
                                isinstance(
1183
                                    delta_message.tool_calls[0].function,
1184
1185
1186
1187
1188
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1189
                                latest_delta_len = len(
1190
1191
                                    delta_message.tool_calls[0].function.arguments
                                )
1192

1193
1194
1195
1196
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1197
1198
1199
1200
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1201

1202
                            # get what we've streamed so far for arguments
1203
                            # for the current tool
1204
1205
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1206
                                actual_call = actual_call[:-latest_delta_len]
1207
1208

                            # check to see if there's anything left to stream
1209
                            remaining_call = expected_call.replace(actual_call, "", 1)
1210
                            # set that as a delta message
1211
1212
                            delta_message = self._create_remaining_args_delta(
                                delta_message, remaining_call, index
1213
                            )
1214

1215
                        # Send the finish response for each request.n only once
1216
1217
1218
1219
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1220
1221
                        if (
                            auto_tools_called
1222
                            or (tools_streamed[i] and not tool_choice_function_name)
1223
1224
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1225
1226
                            finish_reason_ = "tool_calls"
                        else:
1227
1228
1229
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1230
1231
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1232
                            delta=delta_message,
1233
                            logprobs=logprobs,
1234
                            finish_reason=finish_reason_,
1235
                            stop_reason=output.stop_reason,
1236
1237
1238
1239
1240
1241
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1242

1243
                        finish_reason_sent[i] = True
1244

1245
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1246
1247
1248
1249
1250
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1251
1252
                        model=model_name,
                    )
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1263
                    data = chunk.model_dump_json(exclude_unset=True)
1264
1265
                    yield f"data: {data}\n\n"

1266
1267
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1268
1269
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1270
1271
1272
1273
1274
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1275
1276
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1277
1278
                        cached_tokens=num_cached_tokens
                    )
1279
1280
1281
1282
1283
1284
1285

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1286
1287
1288
1289
1290
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1291
                yield f"data: {final_usage_data}\n\n"
1292

1293
1294
1295
1296
1297
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1298
1299
1300
1301
1302
1303
1304
1305
1306
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1307
1308
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1309
1310
1311
1312
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1313
                        output_token_ids=None,  # Consider also logging all token IDs
1314
1315
1316
1317
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1318

1319
1320
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1321
        except Exception as e:
1322
            logger.exception("Error in chat completion stream generator.")
1323
            data = self.create_streaming_error_response(e)
1324
            yield f"data: {data}\n\n"
1325
1326
1327
1328
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1329
1330
1331
1332
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1333
        model_name: str,
1334
        conversation: list[ConversationMessage],
1335
        tokenizer: TokenizerLike | None,
1336
        request_metadata: RequestResponseMetadata,
1337
    ) -> ErrorResponse | ChatCompletionResponse:
1338
        created_time = int(time.time())
1339
        final_res: RequestOutput | None = None
1340

1341
1342
1343
1344
1345
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1346
        except ValueError as e:
1347
            return self.create_error_response(e)
1348

1349
1350
        assert final_res is not None

1351
        choices: list[ChatCompletionResponseChoice] = []
1352
        if self.tool_call_id_type == "kimi_k2":
1353
1354
1355
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1356

1357
1358
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1359
1360
1361
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1362
            token_ids = output.token_ids
1363
            out_logprobs = output.logprobs
1364
            tool_call_info = None
1365

1366
1367
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1368
                logprobs = self._create_chat_logprobs(
1369
                    token_ids=token_ids,
1370
                    top_logprobs=out_logprobs,
1371
                    num_output_top_logprobs=request.top_logprobs,
1372
                    tokenizer=tokenizer,
1373
                    return_as_token_id=request.return_tokens_as_token_ids,
1374
1375
1376
                )
            else:
                logprobs = None
1377
1378

            if self.use_harmony:
1379
                reasoning, content, _ = parse_chat_output(token_ids)
1380
                if not request.include_reasoning:
1381
                    reasoning = None
1382

1383
                if self.tool_parser is not None:
1384
1385
1386
1387
1388
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1389
1390
1391
1392
1393
1394
1395
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1396
                    content = tool_call_info.content
1397
1398
                    message = ChatMessage(
                        role=role,
1399
                        reasoning=reasoning,
1400
1401
1402
1403
1404
1405
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1406
                        reasoning=reasoning,
1407
1408
                        content=content,
                    )
1409
1410
1411
1412
1413

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1414
1415
1416
1417
1418
1419
1420
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1421
                    stop_reason=output.stop_reason,
1422
1423
1424
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1425
1426
1427
                )
                choices.append(choice_data)
                continue
1428

1429
            if self.reasoning_parser:
1430
                try:
1431
1432
1433
1434
1435
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1436
1437
1438
1439
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
                        chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                    )
1440
1441
1442
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1443
1444
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1445
                reasoning, content = reasoning_parser.extract_reasoning(
1446
1447
                    output.text, request=request
                )
1448
                if not request.include_reasoning:
1449
                    reasoning = None
1450
            else:
1451
                reasoning = None
1452
                content = output.text
1453

1454
            auto_tools_called = False
1455
1456
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1467
1468
1469
1470
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1471
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1472
1473

            # if the request uses tools and specified a tool choice
1474
1475
1476
1477
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1478
                assert tool_calls is not None and len(tool_calls) > 0
1479
1480
                message = ChatMessage(
                    role=role,
1481
                    reasoning=reasoning,
1482
                    content="",
1483
                    tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1484
                )
1485

1486
            elif request.tool_choice and request.tool_choice == "required":
1487
1488
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1489
                for tool_call in tool_calls:
1490
1491
1492
1493
1494
1495
1496
1497
                    tool_call_class_items.append(
                        tool_call_class(
                            id=make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
                                idx=history_tool_call_cnt,
                            ),
                            function=tool_call,
1498
1499
                        )
                    )
1500
                    history_tool_call_cnt += 1
1501
1502
1503
                message = ChatMessage(
                    role=role,
                    content="",
1504
                    tool_calls=tool_call_class_items,
1505
                    reasoning=reasoning,
1506
                )
1507

1508
1509
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1510
            elif not request.tool_choice or request.tool_choice == "none":
1511
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1512
1513

            # handle when there are tools and tool choice is auto
1514
1515
1516
1517
1518
1519
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1520
1521
1522
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1523
1524
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1525
1526
                    message = ChatMessage(
                        role=role,
1527
                        reasoning=reasoning,
1528
1529
1530
1531
1532
1533
1534
1535
                        content=content,
                        tool_calls=[
                            ToolCall(
                                function=tc,
                                type="function",
                            )
                            for tc in tool_calls
                        ],
1536
                    )
1537
1538
1539
1540

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1541
1542
1543
1544
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1545
1546
                    if content and len(content) > 0:
                        ret_content = content
1547
1548
                    message = ChatMessage(
                        role=role,
1549
                        reasoning=reasoning,
1550
1551
                        content=ret_content,
                    )
1552
1553
1554
1555
1556
1557

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1558
1559
                    "completion."
                )
1560
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1561
1562
1563
1564
1565
1566
1567
1568
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1569

1570
1571
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1572
                message=message,
1573
                logprobs=logprobs,
1574
1575
1576
1577
1578
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1579
                stop_reason=output.stop_reason,
1580
1581
1582
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1583
            )
1584
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1585

1586
1587
            choices.append(choice_data)

1588
        if request.echo:
1589
            last_msg_content: str | list[dict[str, str]] = ""
1590
1591
1592
1593
1594
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1595
                last_msg_content = conversation[-1]["content"] or ""
1596
            if isinstance(last_msg_content, list):
1597
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1598
1599

            for choice in choices:
1600
                full_message = last_msg_content + (choice.message.content or "")
1601
1602
                choice.message.content = full_message

1603
        assert final_res.prompt_token_ids is not None
1604
        num_prompt_tokens = len(final_res.prompt_token_ids)
1605
1606
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1607
        num_generated_tokens = sum(
1608
1609
1610
1611
1612
1613
1614
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1615
1616
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1617
1618
                cached_tokens=final_res.num_cached_tokens
            )
1619
1620
1621

        request_metadata.final_usage_info = usage

1622
1623
1624
1625
1626
1627
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1628
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1629
1630
1631
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1632
            kv_transfer_params=final_res.kv_transfer_params,
1633
1634
        )

1635
1636
1637
1638
1639
1640
1641
1642
1643
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1644
1645
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1646
1647
                            tc.function, "arguments"
                        ):
1648
                            tool_call_descriptions.append(
1649
1650
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1651
1652
1653
1654
1655
1656
1657
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1658
                        output_token_ids = final_res.outputs[choice.index].token_ids
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1669
        return response
1670
1671

    def _get_top_logprobs(
1672
1673
        self,
        logprobs: dict[int, Logprob],
1674
        top_logprobs: int | None,
1675
        tokenizer: TokenizerLike | None,
1676
1677
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1678
        return [
1679
            ChatCompletionLogProb(
1680
1681
1682
1683
1684
1685
1686
1687
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1688
1689
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1690
1691
            )
            for i, p in enumerate(logprobs.items())
1692
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1693
1694
1695
1696
1697
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1698
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1699
        tokenizer: TokenizerLike | None,
1700
1701
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1702
1703
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1704
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1705

1706
1707
1708
1709
1710
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1711
1712
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1713
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1714
                if should_return_as_token_id:
1715
                    token = f"token_id:{token_id}"
1716
                else:
1717
1718
1719
1720
1721
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1722
                    token = tokenizer.decode(token_id)
1723

1724
1725
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1726
                        token=token,
1727
                        bytes=list(token.encode("utf-8", errors="replace")),
1728
1729
                    )
                )
1730
            else:
1731
1732
1733
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1734
1735
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1736
                        token=self._get_decoded_token(
1737
1738
1739
                            step_token,
                            token_id,
                            tokenizer,
1740
                            should_return_as_token_id,
1741
1742
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1743
1744
1745
1746
1747
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1748
                        top_logprobs=self._get_top_logprobs(
1749
1750
1751
1752
1753
1754
1755
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1756
1757

        return ChatCompletionLogProbs(content=logprobs_content)
1758

1759
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1760
1761
1762
1763
1764
1765
1766
1767
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1768
1769
1770
1771
1772
1773
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1774
1775
1776

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1777
        delta_message: DeltaMessage | None,
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1789
            output.finish_reason is not None
1790
1791
1792
1793
1794
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1795
1796
1797
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1798

1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
    @staticmethod
    def _create_remaining_args_delta(
        delta_message: DeltaMessage,
        remaining_call: str,
        index: int,
    ) -> DeltaMessage:
        """
        Create a delta message for remaining tool arguments, preserving
        id/type/name from the original delta.
        """
        original_tc = next(
            (tc for tc in delta_message.tool_calls if tc.index == index),
            None,
        )
        original_fn = original_tc.function if original_tc else None
        return DeltaMessage(
            tool_calls=[
                DeltaToolCall(
                    index=index,
                    id=original_tc.id if original_tc else None,
                    type=original_tc.type if original_tc else None,
                    function=DeltaFunctionCall(
                        name=original_fn.name if original_fn else None,
                        arguments=remaining_call,
                    ),
                )
            ]
        )

1828
1829
1830
    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
1831
        should_include_tools: bool = True,
1832
1833
1834
    ):
        messages: list[OpenAIMessage] = []

1835
1836
1837
1838
1839
        # because of issues with pydantic we need to potentially
        # re-serialize the tool_calls field of the request
        # for more info: see comment in `maybe_serialize_tool_calls`
        maybe_serialize_tool_calls(request)

1840
1841
1842
1843
1844
1845
1846
1847
        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1848
            python_description=None,
1849
            with_custom_tools=should_include_tools,
1850
        )
1851
1852
1853
        messages.append(sys_msg)

        # Add developer message.
1854
1855
1856
1857
1858
        if request.tools:
            dev_msg = get_developer_message(
                tools=request.tools if should_include_tools else None
            )
            messages.append(dev_msg)
1859
1860

        # Add user message.
1861
        messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1862
1863
1864

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
1865
        engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1866
1867
1868
1869
1870

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1871
        return messages, [engine_prompt]