serving.py 83.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Any, Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
from partial_json_parser.core.options import Allow
17

18
from vllm.engine.protocol import EngineClient
19
20
21
22
23
24
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
25
from vllm.entrypoints.logger import RequestLogger
26
from vllm.entrypoints.openai.chat_completion.protocol import (
27
28
29
30
31
32
33
34
35
36
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
37
38
)
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
39
    TokenState,
40
41
42
    extract_harmony_streaming_delta,
)
from vllm.entrypoints.openai.engine.protocol import (
43
44
45
46
47
48
49
50
51
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
52
from vllm.entrypoints.openai.engine.serving import (
53
54
55
56
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
57
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
58
59
60
61
62
63
64
65
66
from vllm.entrypoints.openai.parser.harmony_utils import (
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
    parse_chat_inputs_to_harmony_messages,
    parse_chat_output,
    render_for_completion,
)
67
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
68
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
69
from vllm.inputs.data import TokensPrompt
70
from vllm.logger import init_logger
71
from vllm.logprobs import Logprob
72
from vllm.outputs import CompletionOutput, RequestOutput
73
from vllm.sampling_params import BeamSearchParams, SamplingParams
74
75
76
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
    MistralTokenizer,
77
78
79
80
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
81
82
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
83
from vllm.tool_parsers.utils import partial_json_loads
84
from vllm.utils.collection_utils import as_list
85
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
86
87
88
89
90

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
91
92
93
    def __init__(
        self,
        engine_client: EngineClient,
94
        models: OpenAIServingModels,
95
96
        response_role: str,
        *,
97
98
        request_logger: RequestLogger | None,
        chat_template: str | None,
99
        chat_template_content_format: ChatTemplateContentFormatOption,
100
        trust_request_chat_template: bool = False,
101
        return_tokens_as_token_ids: bool = False,
102
        reasoning_parser: str = "",
103
        enable_auto_tools: bool = False,
104
        exclude_tools_when_tool_choice_none: bool = False,
105
        tool_parser: str | None = None,
106
        enable_prompt_tokens_details: bool = False,
107
        enable_force_include_usage: bool = False,
108
        enable_log_outputs: bool = False,
109
        enable_log_deltas: bool = True,
110
        log_error_stack: bool = False,
111
        default_chat_template_kwargs: dict[str, Any] | None = None,
112
    ) -> None:
113
114
115
116
117
118
119
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
120

121
        self.response_role = response_role
122
123
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
124
        self.trust_request_chat_template = trust_request_chat_template
125
        self.default_chat_template_kwargs = default_chat_template_kwargs or {}
126
        self.enable_log_outputs = enable_log_outputs
127
        self.enable_log_deltas = enable_log_deltas
128

129
130
131
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

132
133
134
135
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
136
137
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
138
139
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
140
141
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
142

143
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
144
        self.enable_force_include_usage = enable_force_include_usage
145
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
146
        if self.default_sampling_params:
147
148
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
149
150
151
152
153
154
155
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
156
        else:
157
            self.tool_call_id_type = "random"
158

159
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
160
161
162
163
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
164
165
                get_stop_tokens_for_assistant_actions()
            )
166
167
168
169
170
171
172
173
174
175
176

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

177
178
179
180
181
182
183
184
185
186
187
188
    async def warmup(self) -> None:
        """
        Warm up the chat template processing to avoid first-request latency.

        This method triggers Jinja2 template compilation and content format
        detection that would otherwise happen on the first real request,
        causing increased latency on the first request.
        """
        logger.info("Warming up chat template processing...")
        start_time = time.perf_counter()

        try:
189
            renderer = self.engine_client.renderer
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204

            # Create a minimal dummy request
            dummy_request = ChatCompletionRequest(
                messages=[{"role": "user", "content": "warmup"}],
                model=None,
                max_completion_tokens=1,
            )

            # Call _preprocess_chat to trigger template compilation
            # This forces:
            # 1. Chat template content format detection
            # 2. Jinja2 template compilation
            # 3. Tokenizer initialization for chat
            await self._preprocess_chat(
                dummy_request,
205
                renderer,
206
207
208
209
210
211
212
213
                dummy_request.messages,
                chat_template=self.chat_template,
                chat_template_content_format=self.chat_template_content_format,
                add_generation_prompt=True,
                continue_final_message=False,
                tool_dicts=None,
                documents=None,
                chat_template_kwargs=None,
214
                default_chat_template_kwargs=self.default_chat_template_kwargs,
215
216
217
218
219
220
221
222
223
224
225
                tool_parser=None,
                add_special_tokens=False,
            )

            elapsed = (time.perf_counter() - start_time) * 1000
            logger.info("Chat template warmup completed in %.1fms", elapsed)

        except Exception:
            # Log but don't fail server startup if warmup fails
            logger.exception("Chat template warmup failed")

226
    async def render_chat_request(
227
228
        self,
        request: ChatCompletionRequest,
229
    ) -> tuple[list[ConversationMessage], list[Any]] | ErrorResponse:
230
        """
231
        render chat request by validating and preprocessing inputs.
232

233
234
235
        Returns:
            A tuple of (conversation, engine_prompts) on success,
            or an ErrorResponse on failure.
236
237
238
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
239
            logger.error("Error with model %s", error_check_ret)
240
241
            return error_check_ret

242
243
244
245
246
247
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

248
        try:
249
250
            renderer = self.engine_client.renderer
            tokenizer = renderer.tokenizer
251

252
253
            tool_parser = self.tool_parser

254
            if isinstance(tokenizer, MistralTokenizer):
255
256
257
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
258
                maybe_serialize_tool_calls(request)
259
                truncate_tool_call_ids(request)
260
                validate_request_params(request)
261

262
263
264
            # Check if tool parsing is unavailable (common condition)
            tool_parsing_unavailable = (
                tool_parser is None
265
266
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
267
268
269
270
271
272
            )

            # Validate tool_choice when tool parsing is required but unavailable
            if tool_parsing_unavailable and request.tool_choice not in (
                None,
                "none",
273
            ):
274
275
276
277
278
279
280
281
282
283
284
285
286
                if request.tool_choice == "auto" and not self.enable_auto_tools:
                    # for hf tokenizers, "auto" tools requires
                    # --enable-auto-tool-choice and --tool-call-parser
                    return self.create_error_response(
                        '"auto" tool choice requires '
                        "--enable-auto-tool-choice and --tool-call-parser to be set"
                    )
                elif request.tool_choice != "auto":
                    # "required" or named tool requires tool parser
                    return self.create_error_response(
                        f'tool_choice="{request.tool_choice}" requires '
                        "--tool-call-parser to be set"
                    )
287

288
289
290
291
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
292
293
294
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
295

296
297
            if not self.use_harmony:
                # Common case.
298
299
300
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
301
                    trust_request_chat_template=self.trust_request_chat_template,
302
303
304
                )
                if error_check_ret is not None:
                    return error_check_ret
305
306
307
308

                chat_template_kwargs = request.chat_template_kwargs or {}
                chat_template_kwargs.update(reasoning_effort=request.reasoning_effort)

309
                conversation, engine_prompts = await self._preprocess_chat(
310
                    request,
311
                    renderer,
312
                    request.messages,
313
                    chat_template=request.chat_template or self.chat_template,
314
                    chat_template_content_format=self.chat_template_content_format,
315
316
317
318
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
319
                    chat_template_kwargs=chat_template_kwargs,
320
                    default_chat_template_kwargs=self.default_chat_template_kwargs,
321
322
323
324
325
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
326
327
328
329
                should_include_tools = tool_dicts is not None
                conversation, engine_prompts = self._make_request_with_harmony(
                    request, should_include_tools
                )
330
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
331
            logger.exception("Error in preprocessing prompt inputs")
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
            return self.create_error_response(e)

        return conversation, engine_prompts

    async def create_chat_completion(
        self,
        request: ChatCompletionRequest,
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
        """
        Chat Completion API similar to OpenAI's API.

        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
        Chat Completion API.
        """
        result = await self.render_chat_request(request)
        if isinstance(result, ErrorResponse):
            return result

        conversation, engine_prompts = result
353

354
355
356
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
357
358
359
360
361

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

362
363
364
365
366
367
368
369
370
371
        try:
            lora_request = self._maybe_get_adapters(
                request, supports_default_mm_loras=True
            )

            model_name = self.models.model_name(lora_request)
        except (ValueError, TypeError, RuntimeError) as e:
            logger.exception("Error preparing request components")
            return self.create_error_response(e)

372
373
374
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

375
        # Schedule the request and get the result generator.
376
        generators: list[AsyncGenerator[RequestOutput, None]] = []
377
        try:
378
            for i, engine_prompt in enumerate(engine_prompts):
379
                prompt_text, _, _ = self._get_prompt_components(engine_prompt)
380
381
382
383
384
                # If we are creating sub requests for multiple prompts, ensure that they
                # have unique request ids.
                sub_request_id = (
                    request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                )
385
386
387
388
389
390
391
392

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
393
394
                    default_sampling_params=self.default_sampling_params,
                )
395

396
                sampling_params: SamplingParams | BeamSearchParams
397
398
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
399
400
                        max_tokens, self.default_sampling_params
                    )
401
402
                else:
                    sampling_params = request.to_sampling_params(
403
404
405
406
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
407
408
409
410
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
411

412
                self._log_inputs(
413
                    sub_request_id,
414
                    engine_prompt,
415
416
417
                    params=sampling_params,
                    lora_request=lora_request,
                )
418

419
420
421
422
423
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
424
425

                if isinstance(sampling_params, BeamSearchParams):
426
                    generator = self.beam_search(
427
                        prompt=engine_prompt,
428
                        request_id=sub_request_id,
429
                        params=sampling_params,
430
                        lora_request=lora_request,
431
                        trace_headers=trace_headers,
432
433
                    )
                else:
434
                    engine_request, tokenization_kwargs = await self._process_inputs(
435
                        sub_request_id,
436
437
438
439
440
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
441
                        data_parallel_rank=data_parallel_rank,
442
                    )
443

444
                    generator = self.engine_client.generate(
445
                        engine_request,
446
                        sampling_params,
447
                        sub_request_id,
448
449
450
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
451
452
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
453
                        data_parallel_rank=data_parallel_rank,
454
455
456
                    )

                generators.append(generator)
457
        except ValueError as e:
458
            return self.create_error_response(e)
459

460
        assert len(generators) == 1
461
        (result_generator,) = generators
462

463
        # Streaming response
464
465
        tokenizer = self.renderer.tokenizer

466
467
        if request.stream:
            return self.chat_completion_stream_generator(
468
469
470
471
472
473
474
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
475
            )
476

477
478
        try:
            return await self.chat_completion_full_generator(
479
480
481
482
483
484
485
486
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
487
488
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
489
        except ValueError as e:
490
            return self.create_error_response(e)
491
492
493
494

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
495
        return request.messages[-1]["role"]
496

497
    @staticmethod
498
    def _bracket_level(s: str, opening="{", closing="}") -> int:
499
500
501
502
503
504
505
506
507
508
509
510
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
511
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
512
513
514
515
516
517
518
519
520
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
521
            if c == "{":
522
523
                bracket_level += 1
                passed_zero = bracket_level == 0
524
            elif c == "}":
525
526
527
528
529
530
531
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
532
                if c == ",":
533
534
535
536
537
538
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
539
        current_text: str | None,
540
541
        delta_text: str,
        function_name_returned: bool,
542
543
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
544
545
546
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
547
        try:
548
549
550
551
552
553
            flags = Allow.ALL
            obj, _ = partial_json_loads(current_text, flags)
        except (
            partial_json_parser.core.exceptions.MalformedJSON,
            json.JSONDecodeError,
        ):
554
            logger.debug("not enough tokens to parse into JSON yet")
555
556
557
558
559
560
561
562
563
564
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
565
566
                delta_text, previous_text
            )
567
568
569
570
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
571
572
573
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
574
575
576
577
578
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
579
580
581
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
582
583
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
584
585
                        arguments, previous_text
                    )
586
587
588
589

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
590
                    if finishes_previous_tool and "parameters" not in current_tool_call:
591
592
593
                        current_tool_call = obj[-2]

                    function_name_returned = True
594
595
596
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
597
598
599
600
601
602
603
604
605
606
607
608
609
610
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
611
612
613

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
614
615
                        delta_text, previous_text
                    )
616
617

                    if delta_text != "":
618
619
620
621
622
623
624
625
626
627
628
629
630
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
631
632
633
634
635
                    else:
                        delta_message = None

        return delta_message, function_name_returned

636
    async def chat_completion_stream_generator(
637
638
639
640
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
641
        model_name: str,
642
        conversation: list[ConversationMessage],
643
        tokenizer: TokenizerLike | None,
644
        request_metadata: RequestResponseMetadata,
645
    ) -> AsyncGenerator[str, None]:
646
        created_time = int(time.time())
647
        chunk_object_type: Final = "chat.completion.chunk"
648
        first_iteration = True
649
650

        # Send response for each token for each request.n (index)
651
652
653
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
654
        num_prompt_tokens = 0
655
        num_cached_tokens = None
656
657
        if self.use_harmony:
            harmony_parsers = [
658
                get_streamable_parser_for_assistant() for _ in range(num_choices)
659
            ]
660
661
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
662
663
664
665
666
667
668
669
670

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
671
672
            and self._should_stream_with_auto_tool_parsing(request)
        )
673

674
        all_previous_token_ids: list[list[int]] | None
675
        function_name_returned = [False] * num_choices
676
        if self.tool_call_id_type == "kimi_k2":
677
678
679
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
680

681
682
683
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

684
685
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
686
        if tool_choice_auto or self.reasoning_parser:
687
688
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
689
690
691
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
692
        else:
693
            all_previous_token_ids = None
694

695
        try:
696
            if self.reasoning_parser:
697
698
699
700
701
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

702
703
704
705
706
                # Pass the same chat template kwargs as used in tokenization
                chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
                    request.chat_template_kwargs,
                    self.default_chat_template_kwargs,
                )
707
708
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
709
                    chat_template_kwargs=chat_template_kwargs,  # type: ignore[call-arg]
710
                )
711
712
713
714
715
716
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
717
718
719
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
720
721
722
723
724
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

725
                tool_parsers: list[ToolParser | None] = [
726
727
728
729
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
730
        except Exception as e:
731
            logger.exception("Error in tool parser creation.")
732
            data = self.create_streaming_error_response(e)
733
734
735
736
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

737
        stream_options = request.stream_options
738
739
740
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
741

742
743
        try:
            async for res in result_generator:
744
745
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
746
747
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
748

749
750
751
752
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
753
                    num_cached_tokens = res.num_cached_tokens
754
755
                    # Send first response for each request.n (index) with
                    # the role
756
                    role = self.get_chat_request_role(request)
757
758
759

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
760
                    for i in range(num_choices):
761
762
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
763
764
765
766
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
767
                            logprobs=None,
768
769
                            finish_reason=None,
                        )
770
771

                        # return prompt_token_ids at the first chunk ever
772
773
774
775
776
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
777
                            model=model_name,
778
779
780
781
782
783
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
784

785
786
787
788
789
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
790
791
                                total_tokens=num_prompt_tokens,
                            )
792

793
794
795
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

796
797
                    # Send response to echo the input portion of the
                    # last message
798
                    if request.echo:
799
                        last_msg_content: str | list[dict[str, str]] = ""
800
801
802
803
804
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
805
                            last_msg_content = conversation[-1]["content"] or ""
806
807

                        if last_msg_content:
808
                            for i in range(num_choices):
809
810
811
812
813
814
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
815
816
817
818
819
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
820
821
                                    model=model_name,
                                )
822
823
824
825
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
826
827
                                        total_tokens=num_prompt_tokens,
                                    )
828

829
                                data = chunk.model_dump_json(exclude_unset=True)
830
831
832
833
834
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
835
                    tool_parser = tool_parsers[i]
836
837
838
839

                    if finish_reason_sent[i]:
                        continue

840
                    if request.logprobs and request.top_logprobs is not None:
841
                        assert output.logprobs is not None, "Did not output logprobs"
842
                        logprobs = self._create_chat_logprobs(
843
844
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
845
                            tokenizer=tokenizer,
846
                            num_output_top_logprobs=request.top_logprobs,
847
                            return_as_token_id=request.return_tokens_as_token_ids,
848
849
850
851
                        )
                    else:
                        logprobs = None

852
853
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
854
                        prev_recipient = harmony_parser.current_recipient
855
856
857

                        # Track accumulated content per token with their state
                        token_states: list[TokenState] = []
858
859
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
860
861
862
863
864
865
866
867
868
                            token_delta = harmony_parser.last_content_delta or ""
                            token_states.append(
                                TokenState(
                                    harmony_parser.current_channel,
                                    harmony_parser.current_recipient,
                                    token_delta,
                                )
                            )
                        delta_text = "".join(delta for _, _, delta in token_states)
869
                        cur_channel = harmony_parser.current_channel
870

871
872
873
874
875
                        # handle the case where several tokens where generated at once
                        # including the final token, leading to a delta in the text
                        # but the current channel to be empty (start state)
                        if not cur_channel and delta_text:
                            cur_channel = "final"
876
877
                    else:
                        delta_text = output.text
878

879
880
881
882
883
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
884
885
886
                        # Chunked prefill case, don't return empty chunks
                        continue

887
                    delta_message: DeltaMessage | None
888

889
                    # just update previous_texts and previous_token_ids
890
                    if tool_choice_auto or self.reasoning_parser:
891
892
893
894
895
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
896
897
                        # avoid the None + list error.
                        if previous_token_ids:
898
                            current_token_ids = previous_token_ids + as_list(
899
900
                                output.token_ids
                            )
901
                        else:
902
                            current_token_ids = as_list(output.token_ids)
903

904
                    if self.use_harmony:
905
906
907
                        delta_message, tools_streamed_flag = (
                            extract_harmony_streaming_delta(
                                harmony_parser=harmony_parser,
908
                                token_states=token_states,
909
910
911
912
913
                                prev_recipient=prev_recipient,
                                include_reasoning=request.include_reasoning,
                            )
                        )
                        harmony_tools_streamed[i] |= tools_streamed_flag
914
                    # handle streaming deltas for tools with named tool_choice
915
                    elif tool_choice_function_name:
916
917
918
919
920
921
922
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
923
924
                            assert reasoning_parser is not None
                            delta_message = (
925
                                reasoning_parser.extract_reasoning_streaming(
926
927
928
929
930
931
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
932
933
                                )
                            )
934
935
936
937
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
938
                            # Only keep 'content', remove 'reasoning'.
939
                            if reasoning_parser.is_reasoning_end(
940
941
942
943
944
945
946
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
947
                                reasoning_end_arr[i] = True
948
949
950
951
952
953
954
955
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
956
                            if self.reasoning_parser:
957
958
959
                                delta_text = previous_text + delta_text
                                current_text = ""

960
961
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
962
963
964
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
965
966
                            else:
                                delta_tool_call = DeltaToolCall(
967
                                    id=make_tool_call_id(),
968
969
970
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
971
972
973
974
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
975
976
                                function_name_returned[i] = True

977
978
979
980
981
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
982
                            tools_streamed[i] = True
983

984
985
986
987
988
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
989
990
991
992
993
994
995
996
997
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
                            and res.prompt_token_ids
                            and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        ):
                            reasoning_end_arr[i] = True
998

999
1000
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
1001
                                reasoning_parser.extract_reasoning_streaming(
1002
1003
1004
1005
1006
1007
1008
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
1009
                            )
1010
1011
1012
1013
1014
1015
1016
1017
1018
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

1019
                        else:
1020
                            # either finished reasoning or no reasoning at all
1021
                            content = current_text
1022
1023
1024
1025
1026
1027
1028
1029
1030

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
1031
                            )
1032
1033
1034
1035
1036
1037
1038
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
1039

1040
1041
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
1042
                    elif tool_choice_auto and self.reasoning_parser:
1043
1044
1045
1046
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
1047
                        output_token_ids = as_list(output.token_ids)
1048
                        if not reasoning_end_arr[i]:
1049
1050
1051
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
1052
1053
1054
1055
1056
1057
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
1058
                                reasoning_end_arr[i] = True
1059
                                current_token_ids = output_token_ids
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
1070
1071
                                    )
                                )
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
1089
1090

                        # handle tool calls only after reasoning is done,
1091
                        if reasoning_end_arr[i]:
1092
                            delta_token_ids = output_token_ids
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1103
                            delta_message = tool_parser.extract_tool_calls_streaming(
1104
1105
                                previous_text=previous_text,
                                current_text=current_text,
1106
                                delta_text=delta_text,
1107
1108
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1126
1127
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1128

1129
                    # when only reasoning
1130
                    elif self.reasoning_parser:
1131
1132
1133
1134
1135
1136
1137
                        delta_message = reasoning_parser.extract_reasoning_streaming(
                            previous_text,
                            current_text,
                            delta_text,
                            previous_token_ids,
                            current_token_ids,
                            output.token_ids,
1138
                        )
1139
                    # handle streaming just a content delta
1140
1141
1142
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1143
                    # update the previous values for the next iteration
1144
1145
1146
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1147
1148
1149
1150
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1151
1152
1153
1154
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1155

1156
                    # set the previous values for the next iteration
1157
                    previous_num_tokens[i] += len(output.token_ids)
1158
1159
1160
1161
1162
1163

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1164
1165
1166
1167
1168
1169
1170
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1171
                            continue
1172
                        delta_message = DeltaMessage()
1173

1174
1175
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
1176
                        delta_content_parts = []
1177
                        if delta_message.content:
1178
1179
1180
1181
1182
1183
                            delta_content_parts.append(delta_message.content)
                        if delta_message.reasoning_content:
                            reasoning = delta_message.reasoning_content
                            delta_content_parts.append(f"[reasoning: {reasoning}]")
                        if delta_message.tool_calls:
                            tool_args = "".join(
1184
1185
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1186
1187
                                if tc.function and tc.function.arguments
                            )
1188
1189
                            if tool_args:
                                delta_content_parts.append(f"[tool_calls: {tool_args}]")
1190

1191
1192
                        if delta_content_parts and self.enable_log_deltas:
                            delta_content = " ".join(delta_content_parts)
1193
1194
1195
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1196
                                output_token_ids=as_list(output.token_ids),
1197
1198
1199
1200
1201
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1202
1203
1204
1205
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1206
                            delta=delta_message,
1207
                            logprobs=logprobs,
1208
                            finish_reason=None,
1209
1210
1211
1212
1213
1214
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1215
1216

                    # if the model is finished generating
1217
                    else:
1218
1219
1220
1221
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1222
1223
1224
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1225
                        # only happens if we are NOT using structured outputs
1226
                        auto_tools_called = False
1227
                        if tool_parser:
1228
1229
1230
1231
1232
1233
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1234
1235
1236
                        else:
                            index = 0

1237
1238
1239
1240
1241
1242
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1243
                            latest_delta_len = 0
1244
1245
                            if (
                                isinstance(
1246
                                    delta_message.tool_calls[0].function,
1247
1248
1249
1250
1251
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1252
                                latest_delta_len = len(
1253
1254
                                    delta_message.tool_calls[0].function.arguments
                                )
1255

1256
1257
1258
1259
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1260
1261
1262
1263
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1264

1265
                            # get what we've streamed so far for arguments
1266
                            # for the current tool
1267
1268
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1269
                                actual_call = actual_call[:-latest_delta_len]
1270
1271

                            # check to see if there's anything left to stream
1272
                            remaining_call = expected_call.replace(actual_call, "", 1)
1273
                            # set that as a delta message
1274
1275
                            delta_message = self._create_remaining_args_delta(
                                delta_message, remaining_call, index
1276
                            )
1277

1278
                        # Send the finish response for each request.n only once
1279
1280
1281
1282
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1283
1284
                        if (
                            auto_tools_called
1285
                            or (tools_streamed[i] and not tool_choice_function_name)
1286
1287
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1288
1289
                            finish_reason_ = "tool_calls"
                        else:
1290
1291
1292
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1293
1294
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1295
                            delta=delta_message,
1296
                            logprobs=logprobs,
1297
                            finish_reason=finish_reason_,
1298
                            stop_reason=output.stop_reason,
1299
1300
1301
1302
1303
1304
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1305

1306
                        finish_reason_sent[i] = True
1307

1308
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1309
1310
1311
1312
1313
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1314
1315
                        model=model_name,
                    )
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1326
                    data = chunk.model_dump_json(exclude_unset=True)
1327
1328
                    yield f"data: {data}\n\n"

1329
1330
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1331
1332
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1333
1334
1335
1336
1337
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1338
1339
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1340
1341
                        cached_tokens=num_cached_tokens
                    )
1342
1343
1344
1345
1346
1347
1348

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1349
1350
1351
1352
1353
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1354
                yield f"data: {final_usage_data}\n\n"
1355

1356
1357
1358
1359
1360
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1361
1362
1363
1364
1365
1366
1367
1368
1369
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1370
1371
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1372
1373
1374
1375
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1376
                        output_token_ids=None,  # Consider also logging all token IDs
1377
1378
1379
1380
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1381

1382
1383
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1384
        except Exception as e:
1385
            logger.exception("Error in chat completion stream generator.")
1386
            data = self.create_streaming_error_response(e)
1387
            yield f"data: {data}\n\n"
1388
1389
1390
1391
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1392
1393
1394
1395
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1396
        model_name: str,
1397
        conversation: list[ConversationMessage],
1398
        tokenizer: TokenizerLike | None,
1399
        request_metadata: RequestResponseMetadata,
1400
    ) -> ErrorResponse | ChatCompletionResponse:
1401
        created_time = int(time.time())
1402
        final_res: RequestOutput | None = None
1403

1404
1405
1406
1407
1408
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1409
        except ValueError as e:
1410
            return self.create_error_response(e)
1411

1412
1413
        assert final_res is not None

1414
        choices: list[ChatCompletionResponseChoice] = []
1415
        if self.tool_call_id_type == "kimi_k2":
1416
1417
1418
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1419

1420
1421
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1422
1423
1424
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1425
            token_ids = output.token_ids
1426
            out_logprobs = output.logprobs
1427
            tool_call_info = None
1428

1429
1430
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1431
                logprobs = self._create_chat_logprobs(
1432
                    token_ids=token_ids,
1433
                    top_logprobs=out_logprobs,
1434
                    num_output_top_logprobs=request.top_logprobs,
1435
                    tokenizer=tokenizer,
1436
                    return_as_token_id=request.return_tokens_as_token_ids,
1437
1438
1439
                )
            else:
                logprobs = None
1440
1441

            if self.use_harmony:
1442
                reasoning, content, _ = parse_chat_output(token_ids)
1443
                if not request.include_reasoning:
1444
                    reasoning = None
1445

1446
                if self.tool_parser is not None:
1447
1448
1449
1450
1451
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1452
1453
1454
1455
1456
1457
1458
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1459
                    content = tool_call_info.content
1460
1461
                    message = ChatMessage(
                        role=role,
1462
                        reasoning=reasoning,
1463
1464
1465
1466
1467
1468
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1469
                        reasoning=reasoning,
1470
1471
                        content=content,
                    )
1472
1473
1474
1475
1476

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1477
1478
1479
1480
1481
1482
1483
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1484
                    stop_reason=output.stop_reason,
1485
1486
1487
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1488
1489
1490
                )
                choices.append(choice_data)
                continue
1491

1492
            if self.reasoning_parser:
1493
                try:
1494
1495
1496
1497
1498
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1499
1500
1501
1502
1503
                    # Pass the same chat template kwargs as used in tokenization
                    chat_template_kwargs = self._prepare_extra_chat_template_kwargs(
                        request.chat_template_kwargs,
                        self.default_chat_template_kwargs,
                    )
1504
1505
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
1506
                        chat_template_kwargs=chat_template_kwargs,  # type: ignore[call-arg]
1507
                    )
1508
1509
1510
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1511
1512
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1513
                reasoning, content = reasoning_parser.extract_reasoning(
1514
1515
                    output.text, request=request
                )
1516
                if not request.include_reasoning:
1517
                    reasoning = None
1518
            else:
1519
                reasoning = None
1520
                content = output.text
1521

1522
            auto_tools_called = False
1523
1524
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1535
1536
1537
1538
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1539
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1540
1541

            # if the request uses tools and specified a tool choice
1542
1543
1544
1545
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1546
                assert tool_calls is not None and len(tool_calls) > 0
1547
1548
                message = ChatMessage(
                    role=role,
1549
                    reasoning=reasoning,
1550
                    content="",
1551
                    tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1552
                )
1553

1554
            elif request.tool_choice and request.tool_choice == "required":
1555
1556
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1557
                for tool_call in tool_calls:
1558
1559
1560
1561
1562
1563
1564
1565
                    tool_call_class_items.append(
                        tool_call_class(
                            id=make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
                                idx=history_tool_call_cnt,
                            ),
                            function=tool_call,
1566
1567
                        )
                    )
1568
                    history_tool_call_cnt += 1
1569
1570
1571
                message = ChatMessage(
                    role=role,
                    content="",
1572
                    tool_calls=tool_call_class_items,
1573
                    reasoning=reasoning,
1574
                )
1575

1576
1577
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1578
            elif not request.tool_choice or request.tool_choice == "none":
1579
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1580
1581

            # handle when there are tools and tool choice is auto
1582
1583
1584
1585
1586
1587
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1588
1589
1590
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1591
1592
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1593
1594
                    message = ChatMessage(
                        role=role,
1595
                        reasoning=reasoning,
1596
1597
1598
1599
1600
1601
1602
1603
                        content=content,
                        tool_calls=[
                            ToolCall(
                                function=tc,
                                type="function",
                            )
                            for tc in tool_calls
                        ],
1604
                    )
1605
1606
1607
1608

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1609
1610
1611
1612
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1613
1614
                    if content and len(content) > 0:
                        ret_content = content
1615
1616
                    message = ChatMessage(
                        role=role,
1617
                        reasoning=reasoning,
1618
1619
                        content=ret_content,
                    )
1620
1621
1622
1623
1624
1625

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1626
1627
                    "completion."
                )
1628
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1629
1630
1631
1632
1633
1634
1635
1636
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1637

1638
1639
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1640
                message=message,
1641
                logprobs=logprobs,
1642
1643
1644
1645
1646
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1647
                stop_reason=output.stop_reason,
1648
1649
1650
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1651
            )
1652
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1653

1654
1655
            choices.append(choice_data)

1656
        if request.echo:
1657
            last_msg_content: str | list[dict[str, str]] = ""
1658
1659
1660
1661
1662
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1663
                last_msg_content = conversation[-1]["content"] or ""
1664
            if isinstance(last_msg_content, list):
1665
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1666
1667

            for choice in choices:
1668
                full_message = last_msg_content + (choice.message.content or "")
1669
1670
                choice.message.content = full_message

1671
        assert final_res.prompt_token_ids is not None
1672
        num_prompt_tokens = len(final_res.prompt_token_ids)
1673
1674
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1675
        num_generated_tokens = sum(
1676
1677
1678
1679
1680
1681
1682
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1683
1684
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1685
1686
                cached_tokens=final_res.num_cached_tokens
            )
1687
1688
1689

        request_metadata.final_usage_info = usage

1690
1691
1692
1693
1694
1695
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1696
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1697
1698
1699
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1700
            kv_transfer_params=final_res.kv_transfer_params,
1701
1702
        )

1703
1704
1705
1706
1707
1708
1709
1710
1711
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1712
1713
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1714
1715
                            tc.function, "arguments"
                        ):
1716
                            tool_call_descriptions.append(
1717
1718
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1719
1720
1721
1722
1723
1724
1725
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1726
                        output_token_ids = final_res.outputs[choice.index].token_ids
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1737
        return response
1738
1739

    def _get_top_logprobs(
1740
1741
        self,
        logprobs: dict[int, Logprob],
1742
        top_logprobs: int | None,
1743
        tokenizer: TokenizerLike | None,
1744
1745
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1746
        return [
1747
            ChatCompletionLogProb(
1748
1749
1750
1751
1752
1753
1754
1755
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1756
1757
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1758
1759
            )
            for i, p in enumerate(logprobs.items())
1760
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1761
1762
1763
1764
1765
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1766
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1767
        tokenizer: TokenizerLike | None,
1768
1769
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1770
1771
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1772
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1773

1774
1775
1776
1777
1778
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1779
1780
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1781
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1782
                if should_return_as_token_id:
1783
                    token = f"token_id:{token_id}"
1784
                else:
1785
1786
                    if tokenizer is None:
                        raise ValueError(
1787
                            "Unable to get tokenizer because `skip_tokenizer_init=True`"
1788
1789
                        )

1790
                    token = tokenizer.decode(token_id)
1791

1792
1793
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1794
                        token=token,
1795
                        bytes=list(token.encode("utf-8", errors="replace")),
1796
1797
                    )
                )
1798
            else:
1799
1800
1801
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1802
1803
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1804
                        token=self._get_decoded_token(
1805
1806
1807
                            step_token,
                            token_id,
                            tokenizer,
1808
                            should_return_as_token_id,
1809
1810
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1811
1812
1813
1814
1815
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1816
                        top_logprobs=self._get_top_logprobs(
1817
1818
1819
1820
1821
1822
1823
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1824
1825

        return ChatCompletionLogProbs(content=logprobs_content)
1826

1827
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1828
1829
1830
1831
1832
1833
1834
1835
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1836
1837
1838
1839
1840
1841
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1842
1843
1844

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1845
        delta_message: DeltaMessage | None,
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1857
            output.finish_reason is not None
1858
1859
1860
1861
1862
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1863
1864
1865
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1866

1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
    @staticmethod
    def _create_remaining_args_delta(
        delta_message: DeltaMessage,
        remaining_call: str,
        index: int,
    ) -> DeltaMessage:
        """
        Create a delta message for remaining tool arguments, preserving
        id/type/name from the original delta.
        """
        original_tc = next(
            (tc for tc in delta_message.tool_calls if tc.index == index),
            None,
        )
        original_fn = original_tc.function if original_tc else None
        return DeltaMessage(
            tool_calls=[
                DeltaToolCall(
                    index=index,
                    id=original_tc.id if original_tc else None,
                    type=original_tc.type if original_tc else None,
                    function=DeltaFunctionCall(
                        name=original_fn.name if original_fn else None,
                        arguments=remaining_call,
                    ),
                )
            ]
        )

1896
1897
1898
    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
1899
        should_include_tools: bool = True,
1900
1901
1902
    ):
        messages: list[OpenAIMessage] = []

1903
1904
1905
1906
1907
        # because of issues with pydantic we need to potentially
        # re-serialize the tool_calls field of the request
        # for more info: see comment in `maybe_serialize_tool_calls`
        maybe_serialize_tool_calls(request)

1908
1909
1910
1911
1912
1913
1914
1915
        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1916
            python_description=None,
1917
            with_custom_tools=should_include_tools,
1918
        )
1919
1920
1921
        messages.append(sys_msg)

        # Add developer message.
1922
1923
1924
1925
1926
        if request.tools:
            dev_msg = get_developer_message(
                tools=request.tools if should_include_tools else None
            )
            messages.append(dev_msg)
1927
1928

        # Add user message.
1929
        messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1930
1931
1932

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
1933
        engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1934
1935
1936
1937
1938

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1939
        return messages, [engine_prompt]