serving_chat.py 75.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
from pydantic import TypeAdapter
17

18
from vllm.engine.protocol import EngineClient
19
20
21
22
23
24
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
25
from vllm.entrypoints.harmony_utils import (
26
27
28
29
30
31
32
33
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
    parse_chat_input,
    parse_chat_output,
    render_for_completion,
)
34
from vllm.entrypoints.logger import RequestLogger
35
from vllm.entrypoints.openai.protocol import (
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    FunctionCall,
    FunctionDefinition,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
58
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
59
from vllm.entrypoints.openai.tool_parsers import ToolParser
60
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
61
from vllm.entrypoints.utils import get_max_tokens
62
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
63
from vllm.logger import init_logger
64
from vllm.logprobs import Logprob
65
from vllm.outputs import CompletionOutput, RequestOutput
66
from vllm.sampling_params import BeamSearchParams, SamplingParams
67
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
68
69
70
71
72
from vllm.transformers_utils.tokenizers import (
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
73
from vllm.utils import as_list
74
75
76
77
78

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
79
80
81
    def __init__(
        self,
        engine_client: EngineClient,
82
        models: OpenAIServingModels,
83
84
        response_role: str,
        *,
85
86
        request_logger: RequestLogger | None,
        chat_template: str | None,
87
        chat_template_content_format: ChatTemplateContentFormatOption,
88
        trust_request_chat_template: bool = False,
89
        return_tokens_as_token_ids: bool = False,
90
        reasoning_parser: str = "",
91
        enable_auto_tools: bool = False,
92
        exclude_tools_when_tool_choice_none: bool = False,
93
        tool_parser: str | None = None,
94
        enable_prompt_tokens_details: bool = False,
95
        enable_force_include_usage: bool = False,
96
        enable_log_outputs: bool = False,
97
        log_error_stack: bool = False,
98
    ) -> None:
99
100
101
102
103
104
105
106
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            enable_force_include_usage=enable_force_include_usage,
            log_error_stack=log_error_stack,
        )
107

108
        self.response_role = response_role
109
110
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
111
        self.trust_request_chat_template = trust_request_chat_template
112
        self.enable_log_outputs = enable_log_outputs
113

114
115
116
117
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
118
119
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
120
121
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
122
123
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
124

125
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
126
        self.enable_force_include_usage = enable_force_include_usage
127
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
128
        if self.default_sampling_params:
129
130
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
131
132
133
134
135
136
137
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
138
        else:
139
            self.tool_call_id_type = "random"
140

141
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
142
143
144
145
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
146
147
                get_stop_tokens_for_assistant_actions()
            )
148
149
150
151
152
153
154
155
156
157
158

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

159
    async def create_chat_completion(
160
161
        self,
        request: ChatCompletionRequest,
162
163
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
164
165
        """
        Chat Completion API similar to OpenAI's API.
166

167
168
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
169
        Chat Completion API.
170
171
172
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
173
            logger.error("Error with model %s", error_check_ret)
174
175
            return error_check_ret

176
177
178
179
180
181
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

182
        try:
183
            lora_request = self._maybe_get_adapters(
184
185
                request, supports_default_mm_loras=True
            )
186

187
            model_name = self.models.model_name(lora_request)
188

189
            tokenizer = await self.engine_client.get_tokenizer()
190

191
192
            tool_parser = self.tool_parser

193
            if isinstance(tokenizer, MistralTokenizer):
194
195
196
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
197
                maybe_serialize_tool_calls(request)
198
                truncate_tool_call_ids(request)
199
                validate_request_params(request)
200

201
202
203
204
205
206
            if (
                request.tool_choice == "auto"
                and not (self.enable_auto_tools and tool_parser is not None)
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
            ):
207
208
209
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
210
                    '"auto" tool choice requires '
211
212
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
213

214
215
216
217
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
218
219
220
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
221

222
223
            if not self.use_harmony:
                # Common case.
224
225
226
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
227
                    trust_request_chat_template=self.trust_request_chat_template,
228
229
230
                )
                if error_check_ret is not None:
                    return error_check_ret
231
232
233
234
235
236
237
238
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = await self._preprocess_chat(
                    request,
                    tokenizer,
                    request.messages,
239
                    chat_template=request.chat_template or self.chat_template,
240
                    chat_template_content_format=self.chat_template_content_format,
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = self._make_request_with_harmony(request)
256
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
257
            logger.exception("Error in preprocessing prompt inputs")
258
            return self.create_error_response(f"{e} {e.__cause__}")
259

260
261
262
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
263
264
265
266
267

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

268
        # Schedule the request and get the result generator.
269
        generators: list[AsyncGenerator[RequestOutput, None]] = []
270
        try:
271
            for i, engine_prompt in enumerate(engine_prompts):
272
                prompt_text, _, _ = self._get_prompt_components(request_prompts[i])
273
274
275
276
277
278
279
280

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
281
282
                    default_sampling_params=self.default_sampling_params,
                )
283

284
                sampling_params: SamplingParams | BeamSearchParams
285
286
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
287
288
                        max_tokens, self.default_sampling_params
                    )
289
290
                else:
                    sampling_params = request.to_sampling_params(
291
292
293
294
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
295

296
297
298
299
300
301
                self._log_inputs(
                    request_id,
                    request_prompts[i],
                    params=sampling_params,
                    lora_request=lora_request,
                )
302

303
304
305
306
307
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
308
309

                if isinstance(sampling_params, BeamSearchParams):
310
                    generator = self.beam_search(
311
312
313
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
314
                        lora_request=lora_request,
315
316
                    )
                else:
317
318
319
320
321
322
323
324
                    engine_request, tokenization_kwargs = await self._process_inputs(
                        request_id,
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
                    )
325

326
                    generator = self.engine_client.generate(
327
                        engine_request,
328
329
330
331
332
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
333
334
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
335
336
337
                    )

                generators.append(generator)
338
        except ValueError as e:
339
            # TODO: Use a vllm-specific Validation Error
340
341
            return self.create_error_response(str(e))

342
        assert len(generators) == 1
343
        (result_generator,) = generators
344

345
346
347
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
348
349
350
351
352
353
354
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
355
356
                enable_force_include_usage=self.enable_force_include_usage,
            )
357

358
359
        try:
            return await self.chat_completion_full_generator(
360
361
362
363
364
365
366
367
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
368
369
370
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
371
372
373
374

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
375
        return request.messages[-1]["role"]
376

377
    @staticmethod
378
    def _bracket_level(s: str, opening="{", closing="}") -> int:
379
380
381
382
383
384
385
386
387
388
389
390
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
391
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
392
393
394
395
396
397
398
399
400
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
401
            if c == "{":
402
403
                bracket_level += 1
                passed_zero = bracket_level == 0
404
            elif c == "}":
405
406
407
408
409
410
411
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
412
                if c == ",":
413
414
415
416
417
418
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
419
        current_text: str | None,
420
421
        delta_text: str,
        function_name_returned: bool,
422
423
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
424
425
426
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
427
428
429
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
430
            logger.debug("not enough tokens to parse into JSON yet")
431
432
433
434
435
436
437
438
439
440
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
441
442
                delta_text, previous_text
            )
443
444
445
446
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
447
448
449
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
450
451
452
453
454
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
455
456
457
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
458
459
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
460
461
                        arguments, previous_text
                    )
462
463
464
465

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
466
                    if finishes_previous_tool and "parameters" not in current_tool_call:
467
468
469
                        current_tool_call = obj[-2]

                    function_name_returned = True
470
471
472
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
473
474
475
476
477
478
479
480
481
482
483
484
485
486
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
487
488
489

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
490
491
                        delta_text, previous_text
                    )
492
493

                    if delta_text != "":
494
495
496
497
498
499
500
501
502
503
504
505
506
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
507
508
509
510
511
                    else:
                        delta_message = None

        return delta_message, function_name_returned

512
    async def chat_completion_stream_generator(
513
514
515
516
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
517
        model_name: str,
518
        conversation: list[ConversationMessage],
519
        tokenizer: AnyTokenizer,
520
        request_metadata: RequestResponseMetadata,
521
        enable_force_include_usage: bool,
522
    ) -> AsyncGenerator[str, None]:
523
        created_time = int(time.time())
524
        chunk_object_type: Final = "chat.completion.chunk"
525
        first_iteration = True
526
527

        # Send response for each token for each request.n (index)
528
529
530
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
531
        num_prompt_tokens = 0
532
        num_cached_tokens = None
533
534
        if self.use_harmony:
            harmony_parsers = [
535
                get_streamable_parser_for_assistant() for _ in range(num_choices)
536
            ]
537
538
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
539
540
541
542
543
544
545
546
547

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
548
549
            and self._should_stream_with_auto_tool_parsing(request)
        )
550

551
        all_previous_token_ids: list[list[int]] | None
552
        function_name_returned = [False] * num_choices
553
        if self.tool_call_id_type == "kimi_k2":
554
555
556
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
557

558
559
560
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

561
562
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
563
        if tool_choice_auto or self.reasoning_parser:
564
565
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
566
567
568
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
569
570
        elif request.tool_choice == "required":
            all_previous_token_ids = None
571
        else:
572
            all_previous_token_ids = None
573

574
        try:
575
            if self.reasoning_parser:
576
577
578
579
580
581
582
                reasoning_parser = self.reasoning_parser(tokenizer)
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
583
584
585
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
586
                tool_parsers: list[ToolParser | None] = [
587
588
589
590
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
591
        except Exception as e:
592
            logger.exception("Error in tool parser creation.")
593
594
595
596
597
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

598
599
        stream_options = request.stream_options
        if stream_options:
600
601
602
603
            include_usage = stream_options.include_usage or enable_force_include_usage
            include_continuous_usage = (
                include_usage and stream_options.continuous_usage_stats
            )
604
605
606
        else:
            include_usage, include_continuous_usage = False, False

607
608
        try:
            async for res in result_generator:
609
610
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
611
612
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
613

614
615
616
617
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
618
                    num_cached_tokens = res.num_cached_tokens
619
620
                    # Send first response for each request.n (index) with
                    # the role
621
                    role = self.get_chat_request_role(request)
622
623
624

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
625
                    for i in range(num_choices):
626
627
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
628
629
630
631
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
632
                            logprobs=None,
633
634
                            finish_reason=None,
                        )
635
636

                        # return prompt_token_ids at the first chunk ever
637
638
639
640
641
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
642
                            model=model_name,
643
644
645
646
647
648
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
649

650
651
652
653
654
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
655
656
                                total_tokens=num_prompt_tokens,
                            )
657

658
659
660
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

661
662
                    # Send response to echo the input portion of the
                    # last message
663
                    if request.echo:
664
                        last_msg_content: str | list[dict[str, str]] = ""
665
666
667
668
669
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
670
                            last_msg_content = conversation[-1]["content"] or ""
671
672

                        if last_msg_content:
673
                            for i in range(num_choices):
674
675
676
677
678
679
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
680
681
682
683
684
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
685
686
                                    model=model_name,
                                )
687
688
689
690
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
691
692
                                        total_tokens=num_prompt_tokens,
                                    )
693

694
                                data = chunk.model_dump_json(exclude_unset=True)
695
696
697
698
699
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
700
                    tool_parser = tool_parsers[i]
701
702
703
704

                    if finish_reason_sent[i]:
                        continue

705
                    if request.logprobs and request.top_logprobs is not None:
706
                        assert output.logprobs is not None, "Did not output logprobs"
707
                        logprobs = self._create_chat_logprobs(
708
709
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
710
                            tokenizer=tokenizer,
711
                            num_output_top_logprobs=request.top_logprobs,
712
                            return_as_token_id=request.return_tokens_as_token_ids,
713
714
715
716
                        )
                    else:
                        logprobs = None

717
718
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
719
                        prev_recipient = harmony_parser.current_recipient
720
                        delta_text = ""
721
722
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
723
                            delta_text += harmony_parser.last_content_delta or ""
724
725
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
726
727
                    else:
                        delta_text = output.text
728

729
730
731
732
733
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
734
735
736
                        # Chunked prefill case, don't return empty chunks
                        continue

737
                    delta_message: DeltaMessage | None
738

739
                    # just update previous_texts and previous_token_ids
740
                    if tool_choice_auto or self.reasoning_parser:
741
742
743
744
745
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
746
747
                        # avoid the None + list error.
                        if previous_token_ids:
748
                            current_token_ids = previous_token_ids + as_list(
749
750
                                output.token_ids
                            )
751
                        else:
752
                            current_token_ids = as_list(output.token_ids)
753

754
                    if self.use_harmony:
755
                        if cur_channel == "final":
756
                            delta_message = DeltaMessage(content=delta_text)
757
758
759
                        elif cur_channel == "analysis":
                            if request.include_reasoning:
                                delta_message = DeltaMessage(
760
761
                                    reasoning_content=delta_text
                                )
762
763
                            else:
                                delta_message = None
764
765
766
767
768
                        elif (
                            cur_channel == "commentary"
                            and cur_recipient
                            and cur_recipient.startswith("functions.")
                        ):
769
770
771
                            # Count completed tool calls to determine index
                            base_index = 0
                            for msg in harmony_parser.messages:
772
773
774
775
776
                                if (
                                    msg.channel == "commentary"
                                    and msg.recipient
                                    and msg.recipient.startswith("functions.")
                                ):
777
778
779
                                    base_index += 1

                            if prev_recipient != cur_recipient:
780
781
782
783
784
785
786
787
788
789
790
791
792
793
                                tool_name = cur_recipient.split("functions.", 1)[1]
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            id=make_tool_call_id(),
                                            type="function",
                                            function=DeltaFunctionCall(
                                                name=tool_name,
                                                arguments="",
                                            ),
                                            index=base_index,
                                        )
                                    ]
                                )
794
                            elif delta_text:
795
796
797
798
799
800
801
802
803
804
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            index=base_index,
                                            function=DeltaFunctionCall(
                                                arguments=delta_text
                                            ),
                                        )
                                    ]
                                )
805
806
807
808
809
810
811
                            else:
                                delta_message = None

                            if delta_message is not None:
                                harmony_tools_streamed[i] = True
                        else:
                            delta_message = None
812
                    # handle streaming deltas for tools with named tool_choice
813
                    elif tool_choice_function_name:
814
815
816
817
818
819
820
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
821
822
                            assert reasoning_parser is not None
                            delta_message = (
823
                                reasoning_parser.extract_reasoning_content_streaming(
824
825
826
827
828
829
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
830
831
                                )
                            )
832
833
834
835
836
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Only keep 'content', remove 'reasoning_content'.
837
                            if reasoning_parser.is_reasoning_end(
838
839
840
841
842
843
844
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
845
                                reasoning_end_arr[i] = True
846
847
848
849
850
851
852
853
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
854
                            if self.reasoning_parser:
855
856
857
                                delta_text = previous_text + delta_text
                                current_text = ""

858
859
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
860
861
862
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
863
864
                            else:
                                delta_tool_call = DeltaToolCall(
865
                                    id=make_tool_call_id(),
866
867
868
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
869
870
871
872
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
873
874
                                function_name_returned[i] = True

875
876
877
878
879
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
880
                            tools_streamed[i] = True
881

882
883
884
885
886
887
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]

888
                        if self.reasoning_parser:
889
890
891
                            _, content = reasoning_parser.extract_reasoning_content(
                                current_text, request
                            )
892
893
                        else:
                            content = current_text
894
895
896
                        delta_message, function_name_returned[i] = (
                            self.extract_tool_call_required_streaming(
                                previous_text=previous_text,
897
                                current_text=content,
898
                                delta_text=delta_text,
899
                                function_name_returned=fn_name_returned,
900
901
902
903
904
905
906
907
                                tool_call_idx=history_tool_call_cnt,
                            )
                        )
                        if (
                            delta_message
                            and delta_message.tool_calls
                            and delta_message.tool_calls[0].id is not None
                        ):
908
                            history_tool_call_cnt += 1
909
                            tools_streamed[i] = True
910

911
912
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
913
                    elif tool_choice_auto and self.reasoning_parser:
914
915
916
917
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
918
                        output_token_ids = as_list(output.token_ids)
919
920
                        if not reasoning_end_arr[i]:
                            delta_message = (
921
                                reasoning_parser.extract_reasoning_content_streaming(
922
923
924
925
926
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
927
                                    output_token_ids,
928
929
                                )
                            )
930
931
932
933
934
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
935
936
937
938
939
940
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
941
                                reasoning_end_arr[i] = True
942
                                current_token_ids = output_token_ids
943
944
945
946
947
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
948
949
950
951
                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
952
                            if reasoning_parser.is_reasoning_end(output_token_ids):
953
                                reasoning_end_arr[i] = True
954
                                current_token_ids = (
955
                                    reasoning_parser.extract_content_ids(
956
957
958
                                        output_token_ids
                                    )
                                )
959
960
961
962
963
964
965
966
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
967
                            delta_token_ids = output_token_ids
968
969
970
971
972
973
974
975
976
977
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

978
                            delta_message = tool_parser.extract_tool_calls_streaming(
979
980
                                previous_text=previous_text,
                                current_text=current_text,
981
                                delta_text=delta_text,
982
983
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1001
1002
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1003

1004
                    # when only reasoning
1005
                    elif self.reasoning_parser:
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
                        delta_message = (
                            reasoning_parser.extract_reasoning_content_streaming(
                                previous_text,
                                current_text,
                                delta_text,
                                previous_token_ids,
                                current_token_ids,
                                output.token_ids,
                            )
                        )
1016
                    # handle streaming just a content delta
1017
1018
1019
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1020
                    # update the previous values for the next iteration
1021
1022
1023
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1024
1025
1026
1027
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1028
1029
1030
1031
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1032

1033
                    # set the previous values for the next iteration
1034
                    previous_num_tokens[i] += len(output.token_ids)
1035
1036
1037
1038
1039
1040

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1041
1042
1043
1044
                        if output.finish_reason is None:
                            continue
                        else:
                            delta_message = DeltaMessage()
1045

1046
1047
1048
1049
1050
1051
1052
1053
1054
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1055
1056
                                if tc.function and tc.function.arguments
                            )
1057
1058
1059
1060
1061

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1062
                                output_token_ids=as_list(output.token_ids),
1063
1064
1065
1066
1067
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1068
1069
1070
1071
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1072
                            delta=delta_message,
1073
                            logprobs=logprobs,
1074
                            finish_reason=None,
1075
1076
1077
1078
1079
1080
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1081
1082

                    # if the model is finished generating
1083
                    else:
1084
1085
1086
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1087
                        # only happens if we are NOT using structured outputs
1088
                        auto_tools_called = False
1089
                        if tool_parser:
1090
1091
1092
1093
1094
1095
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1096
1097
1098
                        else:
                            index = 0

1099
1100
1101
1102
1103
1104
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1105
                            latest_delta_len = 0
1106
1107
                            if (
                                isinstance(
1108
                                    delta_message.tool_calls[0].function,
1109
1110
1111
1112
1113
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1114
                                latest_delta_len = len(
1115
1116
                                    delta_message.tool_calls[0].function.arguments
                                )
1117

1118
1119
1120
1121
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1122
1123
1124
1125
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1126

1127
                            # get what we've streamed so far for arguments
1128
                            # for the current tool
1129
1130
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1131
                                actual_call = actual_call[:-latest_delta_len]
1132
1133

                            # check to see if there's anything left to stream
1134
                            remaining_call = expected_call.replace(actual_call, "", 1)
1135
                            # set that as a delta message
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    DeltaToolCall(
                                        index=index,
                                        function=DeltaFunctionCall(
                                            arguments=remaining_call
                                        ).model_dump(exclude_none=True),
                                    )
                                ]
                            )
1146

1147
                        # Send the finish response for each request.n only once
1148
1149
1150
1151
1152
                        if (
                            auto_tools_called
                            or tools_streamed[i]
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1153
1154
                            finish_reason_ = "tool_calls"
                        else:
1155
1156
1157
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1158
1159
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1160
                            delta=delta_message,
1161
                            logprobs=logprobs,
1162
                            finish_reason=finish_reason_,
1163
                            stop_reason=output.stop_reason,
1164
1165
1166
1167
1168
1169
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1170

1171
                        finish_reason_sent[i] = True
1172

1173
1174
1175
1176
1177
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1178
1179
                        model=model_name,
                    )
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1190
                    data = chunk.model_dump_json(exclude_unset=True)
1191
1192
                    yield f"data: {data}\n\n"

1193
1194
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1195
1196
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1197
1198
1199
1200
1201
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1202
1203
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1204
1205
                        cached_tokens=num_cached_tokens
                    )
1206
1207
1208
1209
1210
1211
1212

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1213
1214
1215
1216
1217
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1218
                yield f"data: {final_usage_data}\n\n"
1219

1220
1221
1222
1223
1224
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1225
1226
1227
1228
1229
1230
1231
1232
1233
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1234
1235
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1236
1237
1238
1239
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1240
                        output_token_ids=None,  # Consider also logging all token IDs
1241
1242
1243
1244
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1245

1246
        except Exception as e:
1247
            # TODO: Use a vllm-specific Validation Error
1248
            logger.exception("Error in chat completion stream generator.")
1249
1250
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1251
1252
1253
1254
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1255
1256
1257
1258
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1259
        model_name: str,
1260
        conversation: list[ConversationMessage],
1261
        tokenizer: AnyTokenizer,
1262
        request_metadata: RequestResponseMetadata,
1263
    ) -> ErrorResponse | ChatCompletionResponse:
1264
        created_time = int(time.time())
1265
        final_res: RequestOutput | None = None
1266

1267
1268
1269
1270
1271
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1272
1273
1274
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1275

1276
1277
        assert final_res is not None

1278
        choices: list[ChatCompletionResponseChoice] = []
1279
        if self.tool_call_id_type == "kimi_k2":
1280
1281
1282
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1283

1284
1285
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1286
            token_ids = output.token_ids
1287
            out_logprobs = output.logprobs
1288
            tool_call_info = None
1289

1290
1291
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1292
                logprobs = self._create_chat_logprobs(
1293
                    token_ids=token_ids,
1294
                    top_logprobs=out_logprobs,
1295
                    num_output_top_logprobs=request.top_logprobs,
1296
                    tokenizer=tokenizer,
1297
                    return_as_token_id=request.return_tokens_as_token_ids,
1298
1299
1300
                )
            else:
                logprobs = None
1301
1302

            if self.use_harmony:
1303
1304
1305
1306
                reasoning_content, content, _ = parse_chat_output(token_ids)
                if not request.include_reasoning:
                    reasoning_content = None

1307
1308
1309
1310
1311
1312
1313
1314
                if self.tool_parser is not None:
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1315
                    content = tool_call_info.content
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                    )
1328
1329
1330
1331
1332

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1333
1334
1335
1336
1337
                    finish_reason="tool_calls"
                    if (tool_call_info is not None and tool_call_info.tools_called)
                    else output.finish_reason
                    if output.finish_reason
                    else "stop",
1338
1339
1340
1341
                    stop_reason=output.stop_reason,
                )
                choices.append(choice_data)
                continue
1342

1343
            if self.reasoning_parser:
1344
1345
1346
1347
1348
                try:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1349
1350
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1351
1352
1353
                reasoning_content, content = reasoning_parser.extract_reasoning_content(
                    output.text, request=request
                )
1354
1355
                if not request.include_reasoning:
                    reasoning_content = None
1356
1357
1358
            else:
                reasoning_content = None
                content = output.text
1359

1360
            auto_tools_called = False
1361
1362
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1363
1364
1365
1366
1367
1368
1369
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1370
1371

            # if the request uses tools and specified a tool choice
1372
1373
1374
1375
1376
1377
1378
1379
1380
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
                tool_call_class = (
                    MistralToolCall
                    if isinstance(tokenizer, MistralTokenizer)
                    else ToolCall
                )
1381
1382
                message = ChatMessage(
                    role=role,
1383
                    reasoning_content=reasoning_content,
1384
1385
                    content="",
                    tool_calls=[
1386
1387
1388
1389
1390
1391
                        tool_call_class(
                            function=FunctionCall(
                                name=request.tool_choice.function.name,
                                arguments=content,
                            )
                        )
1392
1393
                    ],
                )
1394

1395
            elif request.tool_choice and request.tool_choice == "required":
1396
1397
1398
1399
1400
                tool_call_class = (
                    MistralToolCall
                    if isinstance(tokenizer, MistralTokenizer)
                    else ToolCall
                )
1401
1402
1403

                # the fields of FunctionDefinition are a superset of the
                # tool call outputs and can be used for parsing
1404
                assert content is not None
1405
1406
1407
                tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(
                    content
                )
1408
1409
1410
                tool_call_ids = []
                for tool_call in tool_calls:
                    tool_call_ids.append(
1411
1412
1413
1414
1415
1416
                        make_tool_call_id(
                            id_type=self.tool_call_id_type,
                            func_name=tool_call.name,
                            idx=history_tool_call_cnt,
                        )
                    )
1417
                    history_tool_call_cnt += 1
1418
1419
1420
1421
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
1422
1423
1424
1425
1426
1427
1428
1429
1430
                        tool_call_class(
                            id=tool_call_ids[i],
                            function=FunctionCall(
                                name=tool_call.name,
                                arguments=json.dumps(
                                    tool_call.parameters, ensure_ascii=False
                                ),
                            ),
                        )
1431
1432
                        for i, tool_call in enumerate(tool_calls)
                    ],
1433
1434
                    reasoning_content=reasoning_content,
                )
1435

1436
1437
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1438
            elif not request.tool_choice or request.tool_choice == "none":
1439
1440
1441
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1442
1443

            # handle when there are tools and tool choice is auto
1444
1445
1446
1447
1448
1449
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1450
1451
1452
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
1453
                    logger.exception("Error in tool parser creation.")
1454
1455
                    return self.create_error_response(str(e))

1456
                tool_call_info = tool_parser.extract_tool_calls(
1457
1458
                    content if content is not None else "", request=request
                )
1459
1460
1461
1462
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
1463
                if tool_call_info.tools_called:
1464
1465
1466
1467
1468
1469
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=tool_call_info.content,
                        tool_calls=tool_call_info.tool_calls,
                    )
1470
1471
1472
1473

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1474
1475
1476
1477
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1478
                    if tool_call_info.content and len(tool_call_info.content) > 0:
1479
                        ret_content = tool_call_info.content
1480
1481
1482
1483
1484
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=ret_content,
                    )
1485
1486
1487
1488
1489
1490

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1491
1492
1493
1494
1495
                    "completion."
                )
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1496

1497
1498
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1499
                message=message,
1500
                logprobs=logprobs,
1501
1502
1503
1504
1505
                finish_reason="tool_calls"
                if auto_tools_called
                else output.finish_reason
                if output.finish_reason
                else "stop",
1506
                stop_reason=output.stop_reason,
1507
1508
1509
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1510
            )
1511

1512
1513
            choices.append(choice_data)

1514
        if request.echo:
1515
            last_msg_content: str | list[dict[str, str]] = ""
1516
1517
1518
1519
1520
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1521
                last_msg_content = conversation[-1]["content"] or ""
1522
            if isinstance(last_msg_content, list):
1523
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1524
1525

            for choice in choices:
1526
                full_message = last_msg_content + (choice.message.content or "")
1527
1528
                choice.message.content = full_message

1529
        assert final_res.prompt_token_ids is not None
1530
        num_prompt_tokens = len(final_res.prompt_token_ids)
1531
1532
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1533
        num_generated_tokens = sum(
1534
1535
1536
1537
1538
1539
1540
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1541
1542
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1543
1544
                cached_tokens=final_res.num_cached_tokens
            )
1545
1546
1547

        request_metadata.final_usage_info = usage

1548
1549
1550
1551
1552
1553
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1554
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1555
1556
1557
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1558
            kv_transfer_params=final_res.kv_transfer_params,
1559
1560
        )

1561
1562
1563
1564
1565
1566
1567
1568
1569
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1570
1571
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1572
1573
                            tc.function, "arguments"
                        ):
1574
                            tool_call_descriptions.append(
1575
1576
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1577
1578
1579
1580
1581
1582
1583
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1584
                        output_token_ids = final_res.outputs[choice.index].token_ids
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1595
        return response
1596
1597

    def _get_top_logprobs(
1598
1599
        self,
        logprobs: dict[int, Logprob],
1600
        top_logprobs: int | None,
1601
1602
1603
        tokenizer: AnyTokenizer,
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1604
        return [
1605
            ChatCompletionLogProb(
1606
1607
1608
1609
1610
1611
1612
1613
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1614
1615
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1616
1617
            )
            for i, p in enumerate(logprobs.items())
1618
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1619
1620
1621
1622
1623
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1624
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1625
        tokenizer: AnyTokenizer,
1626
1627
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1628
1629
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1630
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1631

1632
1633
1634
1635
1636
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1637
1638
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1639
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1640
                if should_return_as_token_id:
1641
                    token = f"token_id:{token_id}"
1642
1643
                else:
                    token = tokenizer.decode(token_id)
1644

1645
1646
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1647
                        token=token,
1648
                        bytes=list(token.encode("utf-8", errors="replace")),
1649
1650
                    )
                )
1651
            else:
1652
1653
1654
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1655
1656
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1657
                        token=self._get_decoded_token(
1658
1659
1660
                            step_token,
                            token_id,
                            tokenizer,
1661
                            should_return_as_token_id,
1662
1663
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1664
1665
1666
                        bytes=None
                        if step_decoded is None
                        else list(step_decoded.encode("utf-8", errors="replace")),
1667
                        top_logprobs=self._get_top_logprobs(
1668
1669
1670
1671
1672
1673
1674
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1675
1676

        return ChatCompletionLogProbs(content=logprobs_content)
1677

1678
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1679
1680
1681
1682
1683
1684
1685
1686
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1687
1688
1689
1690
1691
1692
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1693
1694
1695

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1696
        delta_message: DeltaMessage | None,
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1708
            output.finish_reason is not None
1709
1710
1711
1712
1713
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1714
1715
1716
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
    ):
        messages: list[OpenAIMessage] = []

        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1732
            python_description=None,
1733
1734
            with_custom_tools=request.tools is not None,
        )
1735
1736
1737
        messages.append(sys_msg)

        # Add developer message.
1738
        dev_msg = get_developer_message(tools=request.tools)
1739
1740
1741
1742
        messages.append(dev_msg)

        # Add user message.
        for chat_msg in request.messages:
1743
            messages.extend(parse_chat_input(chat_msg))
1744
1745
1746
1747

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1748
1749
1750
1751
1752

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1753
        return messages, [prompt_token_ids], [engine_prompt]