serving.py 81.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Any, Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
from partial_json_parser.core.options import Allow
17

18
from vllm.engine.protocol import EngineClient
19
20
21
22
23
24
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
25
from vllm.entrypoints.logger import RequestLogger
26
from vllm.entrypoints.openai.chat_completion.protocol import (
27
28
29
30
31
32
33
34
35
36
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
37
38
39
40
41
)
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
    extract_harmony_streaming_delta,
)
from vllm.entrypoints.openai.engine.protocol import (
42
43
44
45
46
47
48
49
50
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
51
from vllm.entrypoints.openai.engine.serving import (
52
53
54
55
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
56
57
58
59
60
61
62
63
64
from vllm.entrypoints.openai.parser.harmony_utils import (
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
    parse_chat_inputs_to_harmony_messages,
    parse_chat_output,
    render_for_completion,
)
65
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
66
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
67
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
68
from vllm.inputs.data import TokensPrompt
69
from vllm.logger import init_logger
70
from vllm.logprobs import Logprob
71
from vllm.outputs import CompletionOutput, RequestOutput
72
from vllm.sampling_params import BeamSearchParams, SamplingParams
73
74
75
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
    MistralTokenizer,
76
77
78
79
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
80
81
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
82
from vllm.tool_parsers.utils import partial_json_loads
83
from vllm.utils.collection_utils import as_list
84
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
85
86
87
88
89

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
90
91
92
    def __init__(
        self,
        engine_client: EngineClient,
93
        models: OpenAIServingModels,
94
95
        response_role: str,
        *,
96
97
        request_logger: RequestLogger | None,
        chat_template: str | None,
98
        chat_template_content_format: ChatTemplateContentFormatOption,
99
        trust_request_chat_template: bool = False,
100
        return_tokens_as_token_ids: bool = False,
101
        reasoning_parser: str = "",
102
        enable_auto_tools: bool = False,
103
        exclude_tools_when_tool_choice_none: bool = False,
104
        tool_parser: str | None = None,
105
        enable_prompt_tokens_details: bool = False,
106
        enable_force_include_usage: bool = False,
107
        enable_log_outputs: bool = False,
108
        enable_log_deltas: bool = True,
109
        log_error_stack: bool = False,
110
        default_chat_template_kwargs: dict[str, Any] | None = None,
111
    ) -> None:
112
113
114
115
116
117
118
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
119

120
        self.response_role = response_role
121
122
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
123
        self.trust_request_chat_template = trust_request_chat_template
124
        self.default_chat_template_kwargs = default_chat_template_kwargs or {}
125
        self.enable_log_outputs = enable_log_outputs
126
        self.enable_log_deltas = enable_log_deltas
127

128
129
130
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

131
132
133
134
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
135
136
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
137
138
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
139
140
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
141

142
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
143
        self.enable_force_include_usage = enable_force_include_usage
144
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
145
        if self.default_sampling_params:
146
147
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
148
149
150
151
152
153
154
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
155
        else:
156
            self.tool_call_id_type = "random"
157

158
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
159
160
161
162
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
163
164
                get_stop_tokens_for_assistant_actions()
            )
165
166
167
168
169
170
171
172
173
174
175

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
    async def warmup(self) -> None:
        """
        Warm up the chat template processing to avoid first-request latency.

        This method triggers Jinja2 template compilation and content format
        detection that would otherwise happen on the first real request,
        causing increased latency on the first request.
        """
        logger.info("Warming up chat template processing...")
        start_time = time.perf_counter()

        try:
            # Get the tokenizer from the engine
            tokenizer = await self.engine_client.get_tokenizer()

            # Create a minimal dummy request
            dummy_request = ChatCompletionRequest(
                messages=[{"role": "user", "content": "warmup"}],
                model=None,
                max_completion_tokens=1,
            )

            # Call _preprocess_chat to trigger template compilation
            # This forces:
            # 1. Chat template content format detection
            # 2. Jinja2 template compilation
            # 3. Tokenizer initialization for chat
            await self._preprocess_chat(
                dummy_request,
                tokenizer,
                dummy_request.messages,
                chat_template=self.chat_template,
                chat_template_content_format=self.chat_template_content_format,
                add_generation_prompt=True,
                continue_final_message=False,
                tool_dicts=None,
                documents=None,
                chat_template_kwargs=None,
214
                default_chat_template_kwargs=self.default_chat_template_kwargs,
215
216
217
218
219
220
221
222
223
224
225
                tool_parser=None,
                add_special_tokens=False,
            )

            elapsed = (time.perf_counter() - start_time) * 1000
            logger.info("Chat template warmup completed in %.1fms", elapsed)

        except Exception:
            # Log but don't fail server startup if warmup fails
            logger.exception("Chat template warmup failed")

226
    async def create_chat_completion(
227
228
        self,
        request: ChatCompletionRequest,
229
230
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
231
232
        """
        Chat Completion API similar to OpenAI's API.
233

234
235
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
236
        Chat Completion API.
237
238
239
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
240
            logger.error("Error with model %s", error_check_ret)
241
242
            return error_check_ret

243
244
245
246
247
248
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

249
        try:
250
            lora_request = self._maybe_get_adapters(
251
252
                request, supports_default_mm_loras=True
            )
253

254
            model_name = self.models.model_name(lora_request)
255

256
            tokenizer = await self.engine_client.get_tokenizer()
257

258
259
            tool_parser = self.tool_parser

260
            if isinstance(tokenizer, MistralTokenizer):
261
262
263
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
264
                maybe_serialize_tool_calls(request)
265
                truncate_tool_call_ids(request)
266
                validate_request_params(request)
267

268
269
270
            # Check if tool parsing is unavailable (common condition)
            tool_parsing_unavailable = (
                tool_parser is None
271
272
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
273
274
275
276
277
278
            )

            # Validate tool_choice when tool parsing is required but unavailable
            if tool_parsing_unavailable and request.tool_choice not in (
                None,
                "none",
279
            ):
280
281
282
283
284
285
286
287
288
289
290
291
292
                if request.tool_choice == "auto" and not self.enable_auto_tools:
                    # for hf tokenizers, "auto" tools requires
                    # --enable-auto-tool-choice and --tool-call-parser
                    return self.create_error_response(
                        '"auto" tool choice requires '
                        "--enable-auto-tool-choice and --tool-call-parser to be set"
                    )
                elif request.tool_choice != "auto":
                    # "required" or named tool requires tool parser
                    return self.create_error_response(
                        f'tool_choice="{request.tool_choice}" requires '
                        "--tool-call-parser to be set"
                    )
293

294
295
296
297
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
298
299
300
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
301

302
303
            if not self.use_harmony:
                # Common case.
304
305
306
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
307
                    trust_request_chat_template=self.trust_request_chat_template,
308
309
310
                )
                if error_check_ret is not None:
                    return error_check_ret
311
312
313
314

                chat_template_kwargs = request.chat_template_kwargs or {}
                chat_template_kwargs.update(reasoning_effort=request.reasoning_effort)

315
                conversation, engine_prompts = await self._preprocess_chat(
316
317
318
                    request,
                    tokenizer,
                    request.messages,
319
                    chat_template=request.chat_template or self.chat_template,
320
                    chat_template_content_format=self.chat_template_content_format,
321
322
323
324
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
325
                    chat_template_kwargs=chat_template_kwargs,
326
                    default_chat_template_kwargs=self.default_chat_template_kwargs,
327
328
329
330
331
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
332
333
334
335
                should_include_tools = tool_dicts is not None
                conversation, engine_prompts = self._make_request_with_harmony(
                    request, should_include_tools
                )
336
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
337
            logger.exception("Error in preprocessing prompt inputs")
338
            return self.create_error_response(f"{e} {e.__cause__}")
339

340
341
342
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
343
344
345
346
347

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

348
349
350
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

351
        # Schedule the request and get the result generator.
352
        generators: list[AsyncGenerator[RequestOutput, None]] = []
353
        try:
354
            for i, engine_prompt in enumerate(engine_prompts):
355
                prompt_text, _, _ = self._get_prompt_components(engine_prompt)
356
357
358
359
360
                # If we are creating sub requests for multiple prompts, ensure that they
                # have unique request ids.
                sub_request_id = (
                    request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                )
361
362
363
364
365
366
367
368

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
369
370
                    default_sampling_params=self.default_sampling_params,
                )
371

372
                sampling_params: SamplingParams | BeamSearchParams
373
374
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
375
376
                        max_tokens, self.default_sampling_params
                    )
377
378
                else:
                    sampling_params = request.to_sampling_params(
379
380
381
382
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
383
384
385
386
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
387

388
                self._log_inputs(
389
                    sub_request_id,
390
                    engine_prompt,
391
392
393
                    params=sampling_params,
                    lora_request=lora_request,
                )
394

395
396
397
398
399
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
400
401

                if isinstance(sampling_params, BeamSearchParams):
402
                    generator = self.beam_search(
403
                        prompt=engine_prompt,
404
                        request_id=sub_request_id,
405
                        params=sampling_params,
406
                        lora_request=lora_request,
407
                        trace_headers=trace_headers,
408
409
                    )
                else:
410
                    engine_request, tokenization_kwargs = await self._process_inputs(
411
                        sub_request_id,
412
413
414
415
416
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
417
                        data_parallel_rank=data_parallel_rank,
418
                    )
419

420
                    generator = self.engine_client.generate(
421
                        engine_request,
422
                        sampling_params,
423
                        sub_request_id,
424
425
426
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
427
428
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
429
                        data_parallel_rank=data_parallel_rank,
430
431
432
                    )

                generators.append(generator)
433
        except ValueError as e:
434
            return self.create_error_response(e)
435

436
        assert len(generators) == 1
437
        (result_generator,) = generators
438

439
440
441
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
442
443
444
445
446
447
448
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
449
            )
450

451
452
        try:
            return await self.chat_completion_full_generator(
453
454
455
456
457
458
459
460
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
461
462
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
463
        except ValueError as e:
464
            return self.create_error_response(e)
465
466
467
468

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
469
        return request.messages[-1]["role"]
470

471
    @staticmethod
472
    def _bracket_level(s: str, opening="{", closing="}") -> int:
473
474
475
476
477
478
479
480
481
482
483
484
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
485
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
486
487
488
489
490
491
492
493
494
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
495
            if c == "{":
496
497
                bracket_level += 1
                passed_zero = bracket_level == 0
498
            elif c == "}":
499
500
501
502
503
504
505
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
506
                if c == ",":
507
508
509
510
511
512
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
513
        current_text: str | None,
514
515
        delta_text: str,
        function_name_returned: bool,
516
517
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
518
519
520
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
521
        try:
522
523
524
525
526
527
            flags = Allow.ALL
            obj, _ = partial_json_loads(current_text, flags)
        except (
            partial_json_parser.core.exceptions.MalformedJSON,
            json.JSONDecodeError,
        ):
528
            logger.debug("not enough tokens to parse into JSON yet")
529
530
531
532
533
534
535
536
537
538
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
539
540
                delta_text, previous_text
            )
541
542
543
544
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
545
546
547
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
548
549
550
551
552
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
553
554
555
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
556
557
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
558
559
                        arguments, previous_text
                    )
560
561
562
563

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
564
                    if finishes_previous_tool and "parameters" not in current_tool_call:
565
566
567
                        current_tool_call = obj[-2]

                    function_name_returned = True
568
569
570
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
571
572
573
574
575
576
577
578
579
580
581
582
583
584
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
585
586
587

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
588
589
                        delta_text, previous_text
                    )
590
591

                    if delta_text != "":
592
593
594
595
596
597
598
599
600
601
602
603
604
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
605
606
607
608
609
                    else:
                        delta_message = None

        return delta_message, function_name_returned

610
    async def chat_completion_stream_generator(
611
612
613
614
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
615
        model_name: str,
616
        conversation: list[ConversationMessage],
617
        tokenizer: TokenizerLike | None,
618
        request_metadata: RequestResponseMetadata,
619
    ) -> AsyncGenerator[str, None]:
620
        created_time = int(time.time())
621
        chunk_object_type: Final = "chat.completion.chunk"
622
        first_iteration = True
623
624

        # Send response for each token for each request.n (index)
625
626
627
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
628
        num_prompt_tokens = 0
629
        num_cached_tokens = None
630
631
        if self.use_harmony:
            harmony_parsers = [
632
                get_streamable_parser_for_assistant() for _ in range(num_choices)
633
            ]
634
635
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
636
637
638
639
640
641
642
643
644

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
645
646
            and self._should_stream_with_auto_tool_parsing(request)
        )
647

648
        all_previous_token_ids: list[list[int]] | None
649
        function_name_returned = [False] * num_choices
650
        if self.tool_call_id_type == "kimi_k2":
651
652
653
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
654

655
656
657
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

658
659
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
660
        if tool_choice_auto or self.reasoning_parser:
661
662
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
663
664
665
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
666
        else:
667
            all_previous_token_ids = None
668

669
        try:
670
            if self.reasoning_parser:
671
672
673
674
675
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

676
677
678
679
680
                # Pass the same chat template kwargs as used in tokenization
                chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
                    request.chat_template_kwargs,
                    self.default_chat_template_kwargs,
                )
681
682
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
683
                    chat_template_kwargs=chat_template_kwargs,  # type: ignore[call-arg]
684
                )
685
686
687
688
689
690
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
691
692
693
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
694
695
696
697
698
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

699
                tool_parsers: list[ToolParser | None] = [
700
701
702
703
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
704
        except Exception as e:
705
            logger.exception("Error in tool parser creation.")
706
            data = self.create_streaming_error_response(e)
707
708
709
710
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

711
        stream_options = request.stream_options
712
713
714
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
715

716
717
        try:
            async for res in result_generator:
718
719
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
720
721
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
722

723
724
725
726
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
727
                    num_cached_tokens = res.num_cached_tokens
728
729
                    # Send first response for each request.n (index) with
                    # the role
730
                    role = self.get_chat_request_role(request)
731
732
733

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
734
                    for i in range(num_choices):
735
736
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
737
738
739
740
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
741
                            logprobs=None,
742
743
                            finish_reason=None,
                        )
744
745

                        # return prompt_token_ids at the first chunk ever
746
747
748
749
750
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
751
                            model=model_name,
752
753
754
755
756
757
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
758

759
760
761
762
763
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
764
765
                                total_tokens=num_prompt_tokens,
                            )
766

767
768
769
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

770
771
                    # Send response to echo the input portion of the
                    # last message
772
                    if request.echo:
773
                        last_msg_content: str | list[dict[str, str]] = ""
774
775
776
777
778
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
779
                            last_msg_content = conversation[-1]["content"] or ""
780
781

                        if last_msg_content:
782
                            for i in range(num_choices):
783
784
785
786
787
788
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
789
790
791
792
793
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
794
795
                                    model=model_name,
                                )
796
797
798
799
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
800
801
                                        total_tokens=num_prompt_tokens,
                                    )
802

803
                                data = chunk.model_dump_json(exclude_unset=True)
804
805
806
807
808
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
809
                    tool_parser = tool_parsers[i]
810
811
812
813

                    if finish_reason_sent[i]:
                        continue

814
                    if request.logprobs and request.top_logprobs is not None:
815
                        assert output.logprobs is not None, "Did not output logprobs"
816
                        logprobs = self._create_chat_logprobs(
817
818
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
819
                            tokenizer=tokenizer,
820
                            num_output_top_logprobs=request.top_logprobs,
821
                            return_as_token_id=request.return_tokens_as_token_ids,
822
823
824
825
                        )
                    else:
                        logprobs = None

826
827
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
828
                        prev_recipient = harmony_parser.current_recipient
829
                        delta_text = ""
830
831
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
832
                            delta_text += harmony_parser.last_content_delta or ""
833
834
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
835
836
837
838
839
                        # handle the case where several tokens where generated at once
                        # including the final token, leading to a delta in the text
                        # but the current channel to be empty (start state)
                        if not cur_channel and delta_text:
                            cur_channel = "final"
840
841
                    else:
                        delta_text = output.text
842

843
844
845
846
847
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
848
849
850
                        # Chunked prefill case, don't return empty chunks
                        continue

851
                    delta_message: DeltaMessage | None
852

853
                    # just update previous_texts and previous_token_ids
854
                    if tool_choice_auto or self.reasoning_parser:
855
856
857
858
859
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
860
861
                        # avoid the None + list error.
                        if previous_token_ids:
862
                            current_token_ids = previous_token_ids + as_list(
863
864
                                output.token_ids
                            )
865
                        else:
866
                            current_token_ids = as_list(output.token_ids)
867

868
                    if self.use_harmony:
869
870
871
872
873
874
875
876
877
878
879
                        delta_message, tools_streamed_flag = (
                            extract_harmony_streaming_delta(
                                harmony_parser=harmony_parser,
                                cur_channel=cur_channel,
                                cur_recipient=cur_recipient,
                                prev_recipient=prev_recipient,
                                delta_text=delta_text,
                                include_reasoning=request.include_reasoning,
                            )
                        )
                        harmony_tools_streamed[i] |= tools_streamed_flag
880
                    # handle streaming deltas for tools with named tool_choice
881
                    elif tool_choice_function_name:
882
883
884
885
886
887
888
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
889
890
                            assert reasoning_parser is not None
                            delta_message = (
891
                                reasoning_parser.extract_reasoning_streaming(
892
893
894
895
896
897
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
898
899
                                )
                            )
900
901
902
903
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
904
                            # Only keep 'content', remove 'reasoning'.
905
                            if reasoning_parser.is_reasoning_end(
906
907
908
909
910
911
912
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
913
                                reasoning_end_arr[i] = True
914
915
916
917
918
919
920
921
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
922
                            if self.reasoning_parser:
923
924
925
                                delta_text = previous_text + delta_text
                                current_text = ""

926
927
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
928
929
930
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
931
932
                            else:
                                delta_tool_call = DeltaToolCall(
933
                                    id=make_tool_call_id(),
934
935
936
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
937
938
939
940
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
941
942
                                function_name_returned[i] = True

943
944
945
946
947
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
948
                            tools_streamed[i] = True
949

950
951
952
953
954
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
955
956
957
958
959
960
961
962
963
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
                            and res.prompt_token_ids
                            and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        ):
                            reasoning_end_arr[i] = True
964

965
966
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
967
                                reasoning_parser.extract_reasoning_streaming(
968
969
970
971
972
973
974
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
975
                            )
976
977
978
979
980
981
982
983
984
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

985
                        else:
986
                            # either finished reasoning or no reasoning at all
987
                            content = current_text
988
989
990
991
992
993
994
995
996

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
997
                            )
998
999
1000
1001
1002
1003
1004
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
1005

1006
1007
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
1008
                    elif tool_choice_auto and self.reasoning_parser:
1009
1010
1011
1012
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
1013
                        output_token_ids = as_list(output.token_ids)
1014
                        if not reasoning_end_arr[i]:
1015
1016
1017
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
1018
1019
1020
1021
1022
1023
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
1024
                                reasoning_end_arr[i] = True
1025
                                current_token_ids = output_token_ids
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
1036
1037
                                    )
                                )
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
1055
1056

                        # handle tool calls only after reasoning is done,
1057
                        if reasoning_end_arr[i]:
1058
                            delta_token_ids = output_token_ids
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1069
                            delta_message = tool_parser.extract_tool_calls_streaming(
1070
1071
                                previous_text=previous_text,
                                current_text=current_text,
1072
                                delta_text=delta_text,
1073
1074
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1092
1093
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1094

1095
                    # when only reasoning
1096
                    elif self.reasoning_parser:
1097
1098
1099
1100
1101
1102
1103
                        delta_message = reasoning_parser.extract_reasoning_streaming(
                            previous_text,
                            current_text,
                            delta_text,
                            previous_token_ids,
                            current_token_ids,
                            output.token_ids,
1104
                        )
1105
                    # handle streaming just a content delta
1106
1107
1108
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1109
                    # update the previous values for the next iteration
1110
1111
1112
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1113
1114
1115
1116
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1117
1118
1119
1120
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1121

1122
                    # set the previous values for the next iteration
1123
                    previous_num_tokens[i] += len(output.token_ids)
1124
1125
1126
1127
1128
1129

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1130
1131
1132
1133
1134
1135
1136
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1137
                            continue
1138
                        delta_message = DeltaMessage()
1139

1140
1141
1142
1143
1144
1145
1146
1147
1148
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1149
1150
                                if tc.function and tc.function.arguments
                            )
1151

1152
                        if delta_content and self.enable_log_deltas:
1153
1154
1155
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1156
                                output_token_ids=as_list(output.token_ids),
1157
1158
1159
1160
1161
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1162
1163
1164
1165
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1166
                            delta=delta_message,
1167
                            logprobs=logprobs,
1168
                            finish_reason=None,
1169
1170
1171
1172
1173
1174
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1175
1176

                    # if the model is finished generating
1177
                    else:
1178
1179
1180
1181
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1182
1183
1184
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1185
                        # only happens if we are NOT using structured outputs
1186
                        auto_tools_called = False
1187
                        if tool_parser:
1188
1189
1190
1191
1192
1193
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1194
1195
1196
                        else:
                            index = 0

1197
1198
1199
1200
1201
1202
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1203
                            latest_delta_len = 0
1204
1205
                            if (
                                isinstance(
1206
                                    delta_message.tool_calls[0].function,
1207
1208
1209
1210
1211
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1212
                                latest_delta_len = len(
1213
1214
                                    delta_message.tool_calls[0].function.arguments
                                )
1215

1216
1217
1218
1219
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1220
1221
1222
1223
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1224

1225
                            # get what we've streamed so far for arguments
1226
                            # for the current tool
1227
1228
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1229
                                actual_call = actual_call[:-latest_delta_len]
1230
1231

                            # check to see if there's anything left to stream
1232
                            remaining_call = expected_call.replace(actual_call, "", 1)
1233
                            # set that as a delta message
1234
1235
                            delta_message = self._create_remaining_args_delta(
                                delta_message, remaining_call, index
1236
                            )
1237

1238
                        # Send the finish response for each request.n only once
1239
1240
1241
1242
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1243
1244
                        if (
                            auto_tools_called
1245
                            or (tools_streamed[i] and not tool_choice_function_name)
1246
1247
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1248
1249
                            finish_reason_ = "tool_calls"
                        else:
1250
1251
1252
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1253
1254
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1255
                            delta=delta_message,
1256
                            logprobs=logprobs,
1257
                            finish_reason=finish_reason_,
1258
                            stop_reason=output.stop_reason,
1259
1260
1261
1262
1263
1264
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1265

1266
                        finish_reason_sent[i] = True
1267

1268
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1269
1270
1271
1272
1273
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1274
1275
                        model=model_name,
                    )
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1286
                    data = chunk.model_dump_json(exclude_unset=True)
1287
1288
                    yield f"data: {data}\n\n"

1289
1290
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1291
1292
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1293
1294
1295
1296
1297
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1298
1299
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1300
1301
                        cached_tokens=num_cached_tokens
                    )
1302
1303
1304
1305
1306
1307
1308

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1309
1310
1311
1312
1313
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1314
                yield f"data: {final_usage_data}\n\n"
1315

1316
1317
1318
1319
1320
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1321
1322
1323
1324
1325
1326
1327
1328
1329
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1330
1331
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1332
1333
1334
1335
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1336
                        output_token_ids=None,  # Consider also logging all token IDs
1337
1338
1339
1340
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1341

1342
1343
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1344
        except Exception as e:
1345
            logger.exception("Error in chat completion stream generator.")
1346
            data = self.create_streaming_error_response(e)
1347
            yield f"data: {data}\n\n"
1348
1349
1350
1351
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1352
1353
1354
1355
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1356
        model_name: str,
1357
        conversation: list[ConversationMessage],
1358
        tokenizer: TokenizerLike | None,
1359
        request_metadata: RequestResponseMetadata,
1360
    ) -> ErrorResponse | ChatCompletionResponse:
1361
        created_time = int(time.time())
1362
        final_res: RequestOutput | None = None
1363

1364
1365
1366
1367
1368
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1369
        except ValueError as e:
1370
            return self.create_error_response(e)
1371

1372
1373
        assert final_res is not None

1374
        choices: list[ChatCompletionResponseChoice] = []
1375
        if self.tool_call_id_type == "kimi_k2":
1376
1377
1378
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1379

1380
1381
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1382
1383
1384
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1385
            token_ids = output.token_ids
1386
            out_logprobs = output.logprobs
1387
            tool_call_info = None
1388

1389
1390
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1391
                logprobs = self._create_chat_logprobs(
1392
                    token_ids=token_ids,
1393
                    top_logprobs=out_logprobs,
1394
                    num_output_top_logprobs=request.top_logprobs,
1395
                    tokenizer=tokenizer,
1396
                    return_as_token_id=request.return_tokens_as_token_ids,
1397
1398
1399
                )
            else:
                logprobs = None
1400
1401

            if self.use_harmony:
1402
                reasoning, content, _ = parse_chat_output(token_ids)
1403
                if not request.include_reasoning:
1404
                    reasoning = None
1405

1406
                if self.tool_parser is not None:
1407
1408
1409
1410
1411
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1412
1413
1414
1415
1416
1417
1418
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1419
                    content = tool_call_info.content
1420
1421
                    message = ChatMessage(
                        role=role,
1422
                        reasoning=reasoning,
1423
1424
1425
1426
1427
1428
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1429
                        reasoning=reasoning,
1430
1431
                        content=content,
                    )
1432
1433
1434
1435
1436

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1437
1438
1439
1440
1441
1442
1443
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1444
                    stop_reason=output.stop_reason,
1445
1446
1447
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1448
1449
1450
                )
                choices.append(choice_data)
                continue
1451

1452
            if self.reasoning_parser:
1453
                try:
1454
1455
1456
1457
1458
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1459
1460
1461
1462
1463
                    # Pass the same chat template kwargs as used in tokenization
                    chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
                        request.chat_template_kwargs,
                        self.default_chat_template_kwargs,
                    )
1464
1465
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
1466
                        chat_template_kwargs=chat_template_kwargs,  # type: ignore[call-arg]
1467
                    )
1468
1469
1470
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1471
1472
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1473
                reasoning, content = reasoning_parser.extract_reasoning(
1474
1475
                    output.text, request=request
                )
1476
                if not request.include_reasoning:
1477
                    reasoning = None
1478
            else:
1479
                reasoning = None
1480
                content = output.text
1481

1482
            auto_tools_called = False
1483
1484
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1495
1496
1497
1498
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1499
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1500
1501

            # if the request uses tools and specified a tool choice
1502
1503
1504
1505
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1506
                assert tool_calls is not None and len(tool_calls) > 0
1507
1508
                message = ChatMessage(
                    role=role,
1509
                    reasoning=reasoning,
1510
                    content="",
1511
                    tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1512
                )
1513

1514
            elif request.tool_choice and request.tool_choice == "required":
1515
1516
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1517
                for tool_call in tool_calls:
1518
1519
1520
1521
1522
1523
1524
1525
                    tool_call_class_items.append(
                        tool_call_class(
                            id=make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
                                idx=history_tool_call_cnt,
                            ),
                            function=tool_call,
1526
1527
                        )
                    )
1528
                    history_tool_call_cnt += 1
1529
1530
1531
                message = ChatMessage(
                    role=role,
                    content="",
1532
                    tool_calls=tool_call_class_items,
1533
                    reasoning=reasoning,
1534
                )
1535

1536
1537
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1538
            elif not request.tool_choice or request.tool_choice == "none":
1539
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1540
1541

            # handle when there are tools and tool choice is auto
1542
1543
1544
1545
1546
1547
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1548
1549
1550
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1551
1552
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1553
1554
                    message = ChatMessage(
                        role=role,
1555
                        reasoning=reasoning,
1556
1557
1558
1559
1560
1561
1562
1563
                        content=content,
                        tool_calls=[
                            ToolCall(
                                function=tc,
                                type="function",
                            )
                            for tc in tool_calls
                        ],
1564
                    )
1565
1566
1567
1568

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1569
1570
1571
1572
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1573
1574
                    if content and len(content) > 0:
                        ret_content = content
1575
1576
                    message = ChatMessage(
                        role=role,
1577
                        reasoning=reasoning,
1578
1579
                        content=ret_content,
                    )
1580
1581
1582
1583
1584
1585

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1586
1587
                    "completion."
                )
1588
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1589
1590
1591
1592
1593
1594
1595
1596
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1597

1598
1599
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1600
                message=message,
1601
                logprobs=logprobs,
1602
1603
1604
1605
1606
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1607
                stop_reason=output.stop_reason,
1608
1609
1610
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1611
            )
1612
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1613

1614
1615
            choices.append(choice_data)

1616
        if request.echo:
1617
            last_msg_content: str | list[dict[str, str]] = ""
1618
1619
1620
1621
1622
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1623
                last_msg_content = conversation[-1]["content"] or ""
1624
            if isinstance(last_msg_content, list):
1625
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1626
1627

            for choice in choices:
1628
                full_message = last_msg_content + (choice.message.content or "")
1629
1630
                choice.message.content = full_message

1631
        assert final_res.prompt_token_ids is not None
1632
        num_prompt_tokens = len(final_res.prompt_token_ids)
1633
1634
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1635
        num_generated_tokens = sum(
1636
1637
1638
1639
1640
1641
1642
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1643
1644
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1645
1646
                cached_tokens=final_res.num_cached_tokens
            )
1647
1648
1649

        request_metadata.final_usage_info = usage

1650
1651
1652
1653
1654
1655
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1656
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1657
1658
1659
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1660
            kv_transfer_params=final_res.kv_transfer_params,
1661
1662
        )

1663
1664
1665
1666
1667
1668
1669
1670
1671
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1672
1673
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1674
1675
                            tc.function, "arguments"
                        ):
1676
                            tool_call_descriptions.append(
1677
1678
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1679
1680
1681
1682
1683
1684
1685
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1686
                        output_token_ids = final_res.outputs[choice.index].token_ids
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1697
        return response
1698
1699

    def _get_top_logprobs(
1700
1701
        self,
        logprobs: dict[int, Logprob],
1702
        top_logprobs: int | None,
1703
        tokenizer: TokenizerLike | None,
1704
1705
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1706
        return [
1707
            ChatCompletionLogProb(
1708
1709
1710
1711
1712
1713
1714
1715
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1716
1717
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1718
1719
            )
            for i, p in enumerate(logprobs.items())
1720
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1721
1722
1723
1724
1725
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1726
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1727
        tokenizer: TokenizerLike | None,
1728
1729
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1730
1731
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1732
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1733

1734
1735
1736
1737
1738
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1739
1740
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1741
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1742
                if should_return_as_token_id:
1743
                    token = f"token_id:{token_id}"
1744
                else:
1745
1746
1747
1748
1749
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1750
                    token = tokenizer.decode(token_id)
1751

1752
1753
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1754
                        token=token,
1755
                        bytes=list(token.encode("utf-8", errors="replace")),
1756
1757
                    )
                )
1758
            else:
1759
1760
1761
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1762
1763
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1764
                        token=self._get_decoded_token(
1765
1766
1767
                            step_token,
                            token_id,
                            tokenizer,
1768
                            should_return_as_token_id,
1769
1770
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1771
1772
1773
1774
1775
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1776
                        top_logprobs=self._get_top_logprobs(
1777
1778
1779
1780
1781
1782
1783
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1784
1785

        return ChatCompletionLogProbs(content=logprobs_content)
1786

1787
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1788
1789
1790
1791
1792
1793
1794
1795
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1796
1797
1798
1799
1800
1801
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1802
1803
1804

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1805
        delta_message: DeltaMessage | None,
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1817
            output.finish_reason is not None
1818
1819
1820
1821
1822
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1823
1824
1825
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1826

1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
    @staticmethod
    def _create_remaining_args_delta(
        delta_message: DeltaMessage,
        remaining_call: str,
        index: int,
    ) -> DeltaMessage:
        """
        Create a delta message for remaining tool arguments, preserving
        id/type/name from the original delta.
        """
        original_tc = next(
            (tc for tc in delta_message.tool_calls if tc.index == index),
            None,
        )
        original_fn = original_tc.function if original_tc else None
        return DeltaMessage(
            tool_calls=[
                DeltaToolCall(
                    index=index,
                    id=original_tc.id if original_tc else None,
                    type=original_tc.type if original_tc else None,
                    function=DeltaFunctionCall(
                        name=original_fn.name if original_fn else None,
                        arguments=remaining_call,
                    ),
                )
            ]
        )

1856
1857
1858
    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
1859
        should_include_tools: bool = True,
1860
1861
1862
    ):
        messages: list[OpenAIMessage] = []

1863
1864
1865
1866
1867
        # because of issues with pydantic we need to potentially
        # re-serialize the tool_calls field of the request
        # for more info: see comment in `maybe_serialize_tool_calls`
        maybe_serialize_tool_calls(request)

1868
1869
1870
1871
1872
1873
1874
1875
        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1876
            python_description=None,
1877
            with_custom_tools=should_include_tools,
1878
        )
1879
1880
1881
        messages.append(sys_msg)

        # Add developer message.
1882
1883
1884
1885
1886
        if request.tools:
            dev_msg = get_developer_message(
                tools=request.tools if should_include_tools else None
            )
            messages.append(dev_msg)
1887
1888

        # Add user message.
1889
        messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1890
1891
1892

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
1893
        engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1894
1895
1896
1897
1898

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1899
        return messages, [engine_prompt]