serving_chat.py 75 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
from pydantic import TypeAdapter
17

18
from vllm.engine.protocol import EngineClient
19
20
21
22
23
24
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
25
from vllm.entrypoints.harmony_utils import (
26
27
28
29
30
31
32
33
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
    parse_chat_input,
    parse_chat_output,
    render_for_completion,
)
34
from vllm.entrypoints.logger import RequestLogger
35
from vllm.entrypoints.openai.protocol import (
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    FunctionCall,
    FunctionDefinition,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
58
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
59
from vllm.entrypoints.openai.tool_parsers import ToolParser
60
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
61
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
62
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
63
from vllm.logger import init_logger
64
from vllm.logprobs import Logprob
65
from vllm.outputs import CompletionOutput, RequestOutput
66
from vllm.sampling_params import BeamSearchParams, SamplingParams
67
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
68
69
70
71
72
from vllm.transformers_utils.tokenizers import (
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
73
from vllm.utils import as_list
74
75
76
77
78

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
79
80
81
    def __init__(
        self,
        engine_client: EngineClient,
82
        models: OpenAIServingModels,
83
84
        response_role: str,
        *,
85
86
        request_logger: RequestLogger | None,
        chat_template: str | None,
87
        chat_template_content_format: ChatTemplateContentFormatOption,
88
        trust_request_chat_template: bool = False,
89
        return_tokens_as_token_ids: bool = False,
90
        reasoning_parser: str = "",
91
        enable_auto_tools: bool = False,
92
        exclude_tools_when_tool_choice_none: bool = False,
93
        tool_parser: str | None = None,
94
        enable_prompt_tokens_details: bool = False,
95
        enable_force_include_usage: bool = False,
96
        enable_log_outputs: bool = False,
97
        log_error_stack: bool = False,
98
    ) -> None:
99
100
101
102
103
104
105
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
106

107
        self.response_role = response_role
108
109
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
110
        self.trust_request_chat_template = trust_request_chat_template
111
        self.enable_log_outputs = enable_log_outputs
112

113
114
115
116
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
117
118
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
119
120
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
121
122
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
123

124
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
125
        self.enable_force_include_usage = enable_force_include_usage
126
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
127
        if self.default_sampling_params:
128
129
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
130
131
132
133
134
135
136
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
137
        else:
138
            self.tool_call_id_type = "random"
139

140
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
141
142
143
144
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
145
146
                get_stop_tokens_for_assistant_actions()
            )
147
148
149
150
151
152
153
154
155
156
157

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

158
    async def create_chat_completion(
159
160
        self,
        request: ChatCompletionRequest,
161
162
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
163
164
        """
        Chat Completion API similar to OpenAI's API.
165

166
167
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
168
        Chat Completion API.
169
170
171
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
172
            logger.error("Error with model %s", error_check_ret)
173
174
            return error_check_ret

175
176
177
178
179
180
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

181
        try:
182
            lora_request = self._maybe_get_adapters(
183
184
                request, supports_default_mm_loras=True
            )
185

186
            model_name = self.models.model_name(lora_request)
187

188
            tokenizer = await self.engine_client.get_tokenizer()
189

190
191
            tool_parser = self.tool_parser

192
            if isinstance(tokenizer, MistralTokenizer):
193
194
195
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
196
                maybe_serialize_tool_calls(request)
197
                truncate_tool_call_ids(request)
198
                validate_request_params(request)
199

200
201
202
203
204
205
            if (
                request.tool_choice == "auto"
                and not (self.enable_auto_tools and tool_parser is not None)
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
            ):
206
207
208
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
209
                    '"auto" tool choice requires '
210
211
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
212

213
214
215
216
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
217
218
219
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
220

221
222
            if not self.use_harmony:
                # Common case.
223
224
225
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
226
                    trust_request_chat_template=self.trust_request_chat_template,
227
228
229
                )
                if error_check_ret is not None:
                    return error_check_ret
230
231
232
233
234
235
236
237
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = await self._preprocess_chat(
                    request,
                    tokenizer,
                    request.messages,
238
                    chat_template=request.chat_template or self.chat_template,
239
                    chat_template_content_format=self.chat_template_content_format,
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = self._make_request_with_harmony(request)
255
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
256
            logger.exception("Error in preprocessing prompt inputs")
257
            return self.create_error_response(f"{e} {e.__cause__}")
258

259
260
261
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
262
263
264
265
266

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

267
        # Schedule the request and get the result generator.
268
        generators: list[AsyncGenerator[RequestOutput, None]] = []
269
        try:
270
            for i, engine_prompt in enumerate(engine_prompts):
271
                prompt_text, _, _ = self._get_prompt_components(request_prompts[i])
272
273
274
275
276
277
278
279

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
280
281
                    default_sampling_params=self.default_sampling_params,
                )
282

283
                sampling_params: SamplingParams | BeamSearchParams
284
285
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
286
287
                        max_tokens, self.default_sampling_params
                    )
288
289
                else:
                    sampling_params = request.to_sampling_params(
290
291
292
293
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
294

295
296
297
298
299
300
                self._log_inputs(
                    request_id,
                    request_prompts[i],
                    params=sampling_params,
                    lora_request=lora_request,
                )
301

302
303
304
305
306
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
307
308

                if isinstance(sampling_params, BeamSearchParams):
309
                    generator = self.beam_search(
310
311
312
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
313
                        lora_request=lora_request,
314
315
                    )
                else:
316
317
318
319
320
321
322
323
                    engine_request, tokenization_kwargs = await self._process_inputs(
                        request_id,
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
                    )
324

325
                    generator = self.engine_client.generate(
326
                        engine_request,
327
328
329
330
331
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
332
333
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
334
335
336
                    )

                generators.append(generator)
337
        except ValueError as e:
338
            # TODO: Use a vllm-specific Validation Error
339
340
            return self.create_error_response(str(e))

341
        assert len(generators) == 1
342
        (result_generator,) = generators
343

344
345
346
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
347
348
349
350
351
352
353
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
354
            )
355

356
357
        try:
            return await self.chat_completion_full_generator(
358
359
360
361
362
363
364
365
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
366
367
368
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
369
370
371
372

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
373
        return request.messages[-1]["role"]
374

375
    @staticmethod
376
    def _bracket_level(s: str, opening="{", closing="}") -> int:
377
378
379
380
381
382
383
384
385
386
387
388
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
389
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
390
391
392
393
394
395
396
397
398
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
399
            if c == "{":
400
401
                bracket_level += 1
                passed_zero = bracket_level == 0
402
            elif c == "}":
403
404
405
406
407
408
409
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
410
                if c == ",":
411
412
413
414
415
416
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
417
        current_text: str | None,
418
419
        delta_text: str,
        function_name_returned: bool,
420
421
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
422
423
424
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
425
426
427
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
428
            logger.debug("not enough tokens to parse into JSON yet")
429
430
431
432
433
434
435
436
437
438
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
439
440
                delta_text, previous_text
            )
441
442
443
444
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
445
446
447
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
448
449
450
451
452
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
453
454
455
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
456
457
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
458
459
                        arguments, previous_text
                    )
460
461
462
463

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
464
                    if finishes_previous_tool and "parameters" not in current_tool_call:
465
466
467
                        current_tool_call = obj[-2]

                    function_name_returned = True
468
469
470
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
471
472
473
474
475
476
477
478
479
480
481
482
483
484
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
485
486
487

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
488
489
                        delta_text, previous_text
                    )
490
491

                    if delta_text != "":
492
493
494
495
496
497
498
499
500
501
502
503
504
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
505
506
507
508
509
                    else:
                        delta_message = None

        return delta_message, function_name_returned

510
    async def chat_completion_stream_generator(
511
512
513
514
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
515
        model_name: str,
516
        conversation: list[ConversationMessage],
517
        tokenizer: AnyTokenizer,
518
        request_metadata: RequestResponseMetadata,
519
    ) -> AsyncGenerator[str, None]:
520
        created_time = int(time.time())
521
        chunk_object_type: Final = "chat.completion.chunk"
522
        first_iteration = True
523
524

        # Send response for each token for each request.n (index)
525
526
527
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
528
        num_prompt_tokens = 0
529
        num_cached_tokens = None
530
531
        if self.use_harmony:
            harmony_parsers = [
532
                get_streamable_parser_for_assistant() for _ in range(num_choices)
533
            ]
534
535
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
536
537
538
539
540
541
542
543
544

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
545
546
            and self._should_stream_with_auto_tool_parsing(request)
        )
547

548
        all_previous_token_ids: list[list[int]] | None
549
        function_name_returned = [False] * num_choices
550
        if self.tool_call_id_type == "kimi_k2":
551
552
553
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
554

555
556
557
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

558
559
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
560
        if tool_choice_auto or self.reasoning_parser:
561
562
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
563
564
565
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
566
567
        elif request.tool_choice == "required":
            all_previous_token_ids = None
568
        else:
569
            all_previous_token_ids = None
570

571
        try:
572
            if self.reasoning_parser:
573
574
575
576
577
578
579
                reasoning_parser = self.reasoning_parser(tokenizer)
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
580
581
582
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
583
                tool_parsers: list[ToolParser | None] = [
584
585
586
587
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
588
        except Exception as e:
589
            logger.exception("Error in tool parser creation.")
590
591
592
593
594
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

595
        stream_options = request.stream_options
596
597
598
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
599

600
601
        try:
            async for res in result_generator:
602
603
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
604
605
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
606

607
608
609
610
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
611
                    num_cached_tokens = res.num_cached_tokens
612
613
                    # Send first response for each request.n (index) with
                    # the role
614
                    role = self.get_chat_request_role(request)
615
616
617

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
618
                    for i in range(num_choices):
619
620
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
621
622
623
624
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
625
                            logprobs=None,
626
627
                            finish_reason=None,
                        )
628
629

                        # return prompt_token_ids at the first chunk ever
630
631
632
633
634
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
635
                            model=model_name,
636
637
638
639
640
641
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
642

643
644
645
646
647
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
648
649
                                total_tokens=num_prompt_tokens,
                            )
650

651
652
653
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

654
655
                    # Send response to echo the input portion of the
                    # last message
656
                    if request.echo:
657
                        last_msg_content: str | list[dict[str, str]] = ""
658
659
660
661
662
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
663
                            last_msg_content = conversation[-1]["content"] or ""
664
665

                        if last_msg_content:
666
                            for i in range(num_choices):
667
668
669
670
671
672
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
673
674
675
676
677
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
678
679
                                    model=model_name,
                                )
680
681
682
683
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
684
685
                                        total_tokens=num_prompt_tokens,
                                    )
686

687
                                data = chunk.model_dump_json(exclude_unset=True)
688
689
690
691
692
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
693
                    tool_parser = tool_parsers[i]
694
695
696
697

                    if finish_reason_sent[i]:
                        continue

698
                    if request.logprobs and request.top_logprobs is not None:
699
                        assert output.logprobs is not None, "Did not output logprobs"
700
                        logprobs = self._create_chat_logprobs(
701
702
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
703
                            tokenizer=tokenizer,
704
                            num_output_top_logprobs=request.top_logprobs,
705
                            return_as_token_id=request.return_tokens_as_token_ids,
706
707
708
709
                        )
                    else:
                        logprobs = None

710
711
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
712
                        prev_recipient = harmony_parser.current_recipient
713
                        delta_text = ""
714
715
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
716
                            delta_text += harmony_parser.last_content_delta or ""
717
718
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
719
720
                    else:
                        delta_text = output.text
721

722
723
724
725
726
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
727
728
729
                        # Chunked prefill case, don't return empty chunks
                        continue

730
                    delta_message: DeltaMessage | None
731

732
                    # just update previous_texts and previous_token_ids
733
                    if tool_choice_auto or self.reasoning_parser:
734
735
736
737
738
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
739
740
                        # avoid the None + list error.
                        if previous_token_ids:
741
                            current_token_ids = previous_token_ids + as_list(
742
743
                                output.token_ids
                            )
744
                        else:
745
                            current_token_ids = as_list(output.token_ids)
746

747
                    if self.use_harmony:
748
                        if cur_channel == "final":
749
                            delta_message = DeltaMessage(content=delta_text)
750
751
752
                        elif cur_channel == "analysis":
                            if request.include_reasoning:
                                delta_message = DeltaMessage(
753
754
                                    reasoning_content=delta_text
                                )
755
756
                            else:
                                delta_message = None
757
758
759
760
761
                        elif (
                            cur_channel == "commentary"
                            and cur_recipient
                            and cur_recipient.startswith("functions.")
                        ):
762
763
764
                            # Count completed tool calls to determine index
                            base_index = 0
                            for msg in harmony_parser.messages:
765
766
767
768
769
                                if (
                                    msg.channel == "commentary"
                                    and msg.recipient
                                    and msg.recipient.startswith("functions.")
                                ):
770
771
772
                                    base_index += 1

                            if prev_recipient != cur_recipient:
773
774
775
776
777
778
779
780
781
782
783
784
785
786
                                tool_name = cur_recipient.split("functions.", 1)[1]
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            id=make_tool_call_id(),
                                            type="function",
                                            function=DeltaFunctionCall(
                                                name=tool_name,
                                                arguments="",
                                            ),
                                            index=base_index,
                                        )
                                    ]
                                )
787
                            elif delta_text:
788
789
790
791
792
793
794
795
796
797
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            index=base_index,
                                            function=DeltaFunctionCall(
                                                arguments=delta_text
                                            ),
                                        )
                                    ]
                                )
798
799
800
801
802
803
804
                            else:
                                delta_message = None

                            if delta_message is not None:
                                harmony_tools_streamed[i] = True
                        else:
                            delta_message = None
805
                    # handle streaming deltas for tools with named tool_choice
806
                    elif tool_choice_function_name:
807
808
809
810
811
812
813
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
814
815
                            assert reasoning_parser is not None
                            delta_message = (
816
                                reasoning_parser.extract_reasoning_content_streaming(
817
818
819
820
821
822
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
823
824
                                )
                            )
825
826
827
828
829
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Only keep 'content', remove 'reasoning_content'.
830
                            if reasoning_parser.is_reasoning_end(
831
832
833
834
835
836
837
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
838
                                reasoning_end_arr[i] = True
839
840
841
842
843
844
845
846
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
847
                            if self.reasoning_parser:
848
849
850
                                delta_text = previous_text + delta_text
                                current_text = ""

851
852
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
853
854
855
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
856
857
                            else:
                                delta_tool_call = DeltaToolCall(
858
                                    id=make_tool_call_id(),
859
860
861
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
862
863
864
865
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
866
867
                                function_name_returned[i] = True

868
869
870
871
872
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
873
                            tools_streamed[i] = True
874

875
876
877
878
879
880
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]

881
                        if self.reasoning_parser:
882
883
884
                            _, content = reasoning_parser.extract_reasoning_content(
                                current_text, request
                            )
885
886
                        else:
                            content = current_text
887
888
889
                        delta_message, function_name_returned[i] = (
                            self.extract_tool_call_required_streaming(
                                previous_text=previous_text,
890
                                current_text=content,
891
                                delta_text=delta_text,
892
                                function_name_returned=fn_name_returned,
893
894
895
896
897
898
899
900
                                tool_call_idx=history_tool_call_cnt,
                            )
                        )
                        if (
                            delta_message
                            and delta_message.tool_calls
                            and delta_message.tool_calls[0].id is not None
                        ):
901
                            history_tool_call_cnt += 1
902
                            tools_streamed[i] = True
903

904
905
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
906
                    elif tool_choice_auto and self.reasoning_parser:
907
908
909
910
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
911
                        output_token_ids = as_list(output.token_ids)
912
913
                        if not reasoning_end_arr[i]:
                            delta_message = (
914
                                reasoning_parser.extract_reasoning_content_streaming(
915
916
917
918
919
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
920
                                    output_token_ids,
921
922
                                )
                            )
923
924
925
926
927
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
928
929
930
931
932
933
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
934
                                reasoning_end_arr[i] = True
935
                                current_token_ids = output_token_ids
936
937
938
939
940
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
941
942
943
944
                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
945
                            if reasoning_parser.is_reasoning_end(output_token_ids):
946
                                reasoning_end_arr[i] = True
947
                                current_token_ids = (
948
                                    reasoning_parser.extract_content_ids(
949
950
951
                                        output_token_ids
                                    )
                                )
952
953
954
955
956
957
958
959
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
960
                            delta_token_ids = output_token_ids
961
962
963
964
965
966
967
968
969
970
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

971
                            delta_message = tool_parser.extract_tool_calls_streaming(
972
973
                                previous_text=previous_text,
                                current_text=current_text,
974
                                delta_text=delta_text,
975
976
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
994
995
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
996

997
                    # when only reasoning
998
                    elif self.reasoning_parser:
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
                        delta_message = (
                            reasoning_parser.extract_reasoning_content_streaming(
                                previous_text,
                                current_text,
                                delta_text,
                                previous_token_ids,
                                current_token_ids,
                                output.token_ids,
                            )
                        )
1009
                    # handle streaming just a content delta
1010
1011
1012
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1013
                    # update the previous values for the next iteration
1014
1015
1016
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1017
1018
1019
1020
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1021
1022
1023
1024
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1025

1026
                    # set the previous values for the next iteration
1027
                    previous_num_tokens[i] += len(output.token_ids)
1028
1029
1030
1031
1032
1033

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1034
1035
1036
1037
                        if output.finish_reason is None:
                            continue
                        else:
                            delta_message = DeltaMessage()
1038

1039
1040
1041
1042
1043
1044
1045
1046
1047
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1048
1049
                                if tc.function and tc.function.arguments
                            )
1050
1051
1052
1053
1054

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1055
                                output_token_ids=as_list(output.token_ids),
1056
1057
1058
1059
1060
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1061
1062
1063
1064
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1065
                            delta=delta_message,
1066
                            logprobs=logprobs,
1067
                            finish_reason=None,
1068
1069
1070
1071
1072
1073
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1074
1075

                    # if the model is finished generating
1076
                    else:
1077
1078
1079
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1080
                        # only happens if we are NOT using structured outputs
1081
                        auto_tools_called = False
1082
                        if tool_parser:
1083
1084
1085
1086
1087
1088
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1089
1090
1091
                        else:
                            index = 0

1092
1093
1094
1095
1096
1097
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1098
                            latest_delta_len = 0
1099
1100
                            if (
                                isinstance(
1101
                                    delta_message.tool_calls[0].function,
1102
1103
1104
1105
1106
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1107
                                latest_delta_len = len(
1108
1109
                                    delta_message.tool_calls[0].function.arguments
                                )
1110

1111
1112
1113
1114
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1115
1116
1117
1118
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1119

1120
                            # get what we've streamed so far for arguments
1121
                            # for the current tool
1122
1123
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1124
                                actual_call = actual_call[:-latest_delta_len]
1125
1126

                            # check to see if there's anything left to stream
1127
                            remaining_call = expected_call.replace(actual_call, "", 1)
1128
                            # set that as a delta message
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    DeltaToolCall(
                                        index=index,
                                        function=DeltaFunctionCall(
                                            arguments=remaining_call
                                        ).model_dump(exclude_none=True),
                                    )
                                ]
                            )
1139

1140
                        # Send the finish response for each request.n only once
1141
1142
1143
1144
1145
                        if (
                            auto_tools_called
                            or tools_streamed[i]
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1146
1147
                            finish_reason_ = "tool_calls"
                        else:
1148
1149
1150
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1151
1152
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1153
                            delta=delta_message,
1154
                            logprobs=logprobs,
1155
                            finish_reason=finish_reason_,
1156
                            stop_reason=output.stop_reason,
1157
1158
1159
1160
1161
1162
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1163

1164
                        finish_reason_sent[i] = True
1165

1166
1167
1168
1169
1170
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1171
1172
                        model=model_name,
                    )
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1183
                    data = chunk.model_dump_json(exclude_unset=True)
1184
1185
                    yield f"data: {data}\n\n"

1186
1187
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1188
1189
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1190
1191
1192
1193
1194
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1195
1196
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1197
1198
                        cached_tokens=num_cached_tokens
                    )
1199
1200
1201
1202
1203
1204
1205

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1206
1207
1208
1209
1210
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1211
                yield f"data: {final_usage_data}\n\n"
1212

1213
1214
1215
1216
1217
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1218
1219
1220
1221
1222
1223
1224
1225
1226
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1227
1228
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1229
1230
1231
1232
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1233
                        output_token_ids=None,  # Consider also logging all token IDs
1234
1235
1236
1237
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1238

1239
        except Exception as e:
1240
            # TODO: Use a vllm-specific Validation Error
1241
            logger.exception("Error in chat completion stream generator.")
1242
1243
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1244
1245
1246
1247
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1248
1249
1250
1251
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1252
        model_name: str,
1253
        conversation: list[ConversationMessage],
1254
        tokenizer: AnyTokenizer,
1255
        request_metadata: RequestResponseMetadata,
1256
    ) -> ErrorResponse | ChatCompletionResponse:
1257
        created_time = int(time.time())
1258
        final_res: RequestOutput | None = None
1259

1260
1261
1262
1263
1264
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1265
1266
1267
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1268

1269
1270
        assert final_res is not None

1271
        choices: list[ChatCompletionResponseChoice] = []
1272
        if self.tool_call_id_type == "kimi_k2":
1273
1274
1275
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1276

1277
1278
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1279
            token_ids = output.token_ids
1280
            out_logprobs = output.logprobs
1281
            tool_call_info = None
1282

1283
1284
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1285
                logprobs = self._create_chat_logprobs(
1286
                    token_ids=token_ids,
1287
                    top_logprobs=out_logprobs,
1288
                    num_output_top_logprobs=request.top_logprobs,
1289
                    tokenizer=tokenizer,
1290
                    return_as_token_id=request.return_tokens_as_token_ids,
1291
1292
1293
                )
            else:
                logprobs = None
1294
1295

            if self.use_harmony:
1296
1297
1298
1299
                reasoning_content, content, _ = parse_chat_output(token_ids)
                if not request.include_reasoning:
                    reasoning_content = None

1300
1301
1302
1303
1304
1305
1306
1307
                if self.tool_parser is not None:
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1308
                    content = tool_call_info.content
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                    )
1321
1322
1323
1324
1325

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1326
1327
1328
1329
1330
                    finish_reason="tool_calls"
                    if (tool_call_info is not None and tool_call_info.tools_called)
                    else output.finish_reason
                    if output.finish_reason
                    else "stop",
1331
1332
1333
1334
                    stop_reason=output.stop_reason,
                )
                choices.append(choice_data)
                continue
1335

1336
            if self.reasoning_parser:
1337
1338
1339
1340
1341
                try:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1342
1343
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1344
1345
1346
                reasoning_content, content = reasoning_parser.extract_reasoning_content(
                    output.text, request=request
                )
1347
1348
                if not request.include_reasoning:
                    reasoning_content = None
1349
1350
1351
            else:
                reasoning_content = None
                content = output.text
1352

1353
            auto_tools_called = False
1354
1355
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1356
1357
1358
1359
1360
1361
1362
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1363
1364

            # if the request uses tools and specified a tool choice
1365
1366
1367
1368
1369
1370
1371
1372
1373
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
                tool_call_class = (
                    MistralToolCall
                    if isinstance(tokenizer, MistralTokenizer)
                    else ToolCall
                )
1374
1375
                message = ChatMessage(
                    role=role,
1376
                    reasoning_content=reasoning_content,
1377
1378
                    content="",
                    tool_calls=[
1379
1380
1381
1382
1383
1384
                        tool_call_class(
                            function=FunctionCall(
                                name=request.tool_choice.function.name,
                                arguments=content,
                            )
                        )
1385
1386
                    ],
                )
1387

1388
            elif request.tool_choice and request.tool_choice == "required":
1389
1390
1391
1392
1393
                tool_call_class = (
                    MistralToolCall
                    if isinstance(tokenizer, MistralTokenizer)
                    else ToolCall
                )
1394
1395
1396

                # the fields of FunctionDefinition are a superset of the
                # tool call outputs and can be used for parsing
1397
                assert content is not None
1398
1399
1400
                tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
                    content
                )
1401
1402
1403
                tool_call_ids = []
                for tool_call in tool_calls:
                    tool_call_ids.append(
1404
1405
1406
1407
1408
1409
                        make_tool_call_id(
                            id_type=self.tool_call_id_type,
                            func_name=tool_call.name,
                            idx=history_tool_call_cnt,
                        )
                    )
1410
                    history_tool_call_cnt += 1
1411
1412
1413
1414
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
1415
1416
1417
1418
1419
1420
1421
1422
1423
                        tool_call_class(
                            id=tool_call_ids[i],
                            function=FunctionCall(
                                name=tool_call.name,
                                arguments=json.dumps(
                                    tool_call.parameters, ensure_ascii=False
                                ),
                            ),
                        )
1424
1425
                        for i, tool_call in enumerate(tool_calls)
                    ],
1426
1427
                    reasoning_content=reasoning_content,
                )
1428

1429
1430
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1431
            elif not request.tool_choice or request.tool_choice == "none":
1432
1433
1434
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1435
1436

            # handle when there are tools and tool choice is auto
1437
1438
1439
1440
1441
1442
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1443
1444
1445
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
1446
                    logger.exception("Error in tool parser creation.")
1447
1448
                    return self.create_error_response(str(e))

1449
                tool_call_info = tool_parser.extract_tool_calls(
1450
1451
                    content if content is not None else "", request=request
                )
1452
1453
1454
1455
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
1456
                if tool_call_info.tools_called:
1457
1458
1459
1460
1461
1462
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=tool_call_info.content,
                        tool_calls=tool_call_info.tool_calls,
                    )
1463
1464
1465
1466

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1467
1468
1469
1470
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1471
                    if tool_call_info.content and len(tool_call_info.content) > 0:
1472
                        ret_content = tool_call_info.content
1473
1474
1475
1476
1477
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=ret_content,
                    )
1478
1479
1480
1481
1482
1483

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1484
1485
1486
1487
1488
                    "completion."
                )
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1489

1490
1491
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1492
                message=message,
1493
                logprobs=logprobs,
1494
1495
1496
1497
1498
                finish_reason="tool_calls"
                if auto_tools_called
                else output.finish_reason
                if output.finish_reason
                else "stop",
1499
                stop_reason=output.stop_reason,
1500
1501
1502
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1503
            )
1504

1505
1506
            choices.append(choice_data)

1507
        if request.echo:
1508
            last_msg_content: str | list[dict[str, str]] = ""
1509
1510
1511
1512
1513
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1514
                last_msg_content = conversation[-1]["content"] or ""
1515
            if isinstance(last_msg_content, list):
1516
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1517
1518

            for choice in choices:
1519
                full_message = last_msg_content + (choice.message.content or "")
1520
1521
                choice.message.content = full_message

1522
        assert final_res.prompt_token_ids is not None
1523
        num_prompt_tokens = len(final_res.prompt_token_ids)
1524
1525
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1526
        num_generated_tokens = sum(
1527
1528
1529
1530
1531
1532
1533
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1534
1535
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1536
1537
                cached_tokens=final_res.num_cached_tokens
            )
1538
1539
1540

        request_metadata.final_usage_info = usage

1541
1542
1543
1544
1545
1546
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1547
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1548
1549
1550
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1551
            kv_transfer_params=final_res.kv_transfer_params,
1552
1553
        )

1554
1555
1556
1557
1558
1559
1560
1561
1562
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1563
1564
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1565
1566
                            tc.function, "arguments"
                        ):
1567
                            tool_call_descriptions.append(
1568
1569
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1570
1571
1572
1573
1574
1575
1576
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1577
                        output_token_ids = final_res.outputs[choice.index].token_ids
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1588
        return response
1589
1590

    def _get_top_logprobs(
1591
1592
        self,
        logprobs: dict[int, Logprob],
1593
        top_logprobs: int | None,
1594
1595
1596
        tokenizer: AnyTokenizer,
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1597
        return [
1598
            ChatCompletionLogProb(
1599
1600
1601
1602
1603
1604
1605
1606
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1607
1608
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1609
1610
            )
            for i, p in enumerate(logprobs.items())
1611
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1612
1613
1614
1615
1616
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1617
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1618
        tokenizer: AnyTokenizer,
1619
1620
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1621
1622
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1623
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1624

1625
1626
1627
1628
1629
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1630
1631
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1632
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1633
                if should_return_as_token_id:
1634
                    token = f"token_id:{token_id}"
1635
1636
                else:
                    token = tokenizer.decode(token_id)
1637

1638
1639
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1640
                        token=token,
1641
                        bytes=list(token.encode("utf-8", errors="replace")),
1642
1643
                    )
                )
1644
            else:
1645
1646
1647
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1648
1649
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1650
                        token=self._get_decoded_token(
1651
1652
1653
                            step_token,
                            token_id,
                            tokenizer,
1654
                            should_return_as_token_id,
1655
1656
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1657
1658
1659
                        bytes=None
                        if step_decoded is None
                        else list(step_decoded.encode("utf-8", errors="replace")),
1660
                        top_logprobs=self._get_top_logprobs(
1661
1662
1663
1664
1665
1666
1667
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1668
1669

        return ChatCompletionLogProbs(content=logprobs_content)
1670

1671
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1672
1673
1674
1675
1676
1677
1678
1679
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1680
1681
1682
1683
1684
1685
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1686
1687
1688

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1689
        delta_message: DeltaMessage | None,
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1701
            output.finish_reason is not None
1702
1703
1704
1705
1706
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1707
1708
1709
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
    ):
        messages: list[OpenAIMessage] = []

        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1725
            python_description=None,
1726
1727
            with_custom_tools=request.tools is not None,
        )
1728
1729
1730
        messages.append(sys_msg)

        # Add developer message.
1731
        dev_msg = get_developer_message(tools=request.tools)
1732
1733
1734
1735
        messages.append(dev_msg)

        # Add user message.
        for chat_msg in request.messages:
1736
            messages.extend(parse_chat_input(chat_msg))
1737
1738
1739
1740

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1741
1742
1743
1744
1745

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1746
        return messages, [prompt_token_ids], [engine_prompt]