serving_chat.py 76.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
22
23
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
24
from vllm.entrypoints.harmony_utils import (
25
26
27
28
29
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
    parse_chat_output,
30
    parse_input_to_harmony_message,
31
32
    render_for_completion,
)
33
from vllm.entrypoints.logger import RequestLogger
34
from vllm.entrypoints.openai.protocol import (
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
55
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
56
from vllm.entrypoints.openai.tool_parsers import ToolParser
57
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
58
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
59
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
60
from vllm.logger import init_logger
61
from vllm.logprobs import Logprob
62
from vllm.outputs import CompletionOutput, RequestOutput
63
from vllm.sampling_params import BeamSearchParams, SamplingParams
64
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
65
66
67
68
69
from vllm.transformers_utils.tokenizers import (
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
70
from vllm.utils.collection_utils import as_list
71
72
73
74
75

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
76
77
78
    def __init__(
        self,
        engine_client: EngineClient,
79
        models: OpenAIServingModels,
80
81
        response_role: str,
        *,
82
83
        request_logger: RequestLogger | None,
        chat_template: str | None,
84
        chat_template_content_format: ChatTemplateContentFormatOption,
85
        trust_request_chat_template: bool = False,
86
        return_tokens_as_token_ids: bool = False,
87
        reasoning_parser: str = "",
88
        enable_auto_tools: bool = False,
89
        exclude_tools_when_tool_choice_none: bool = False,
90
        tool_parser: str | None = None,
91
        enable_prompt_tokens_details: bool = False,
92
        enable_force_include_usage: bool = False,
93
        enable_log_outputs: bool = False,
94
        log_error_stack: bool = False,
95
    ) -> None:
96
97
98
99
100
101
102
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
103

104
        self.response_role = response_role
105
106
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
107
        self.trust_request_chat_template = trust_request_chat_template
108
        self.enable_log_outputs = enable_log_outputs
109

110
111
112
113
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
114
115
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
116
117
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
118
119
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
120

121
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
122
        self.enable_force_include_usage = enable_force_include_usage
123
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
124
        if self.default_sampling_params:
125
126
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
127
128
129
130
131
132
133
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
134
        else:
135
            self.tool_call_id_type = "random"
136

137
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
138
139
140
141
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
142
143
                get_stop_tokens_for_assistant_actions()
            )
144
145
146
147
148
149
150
151
152
153
154

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

155
    async def create_chat_completion(
156
157
        self,
        request: ChatCompletionRequest,
158
159
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
160
161
        """
        Chat Completion API similar to OpenAI's API.
162

163
164
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
165
        Chat Completion API.
166
167
168
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
169
            logger.error("Error with model %s", error_check_ret)
170
171
            return error_check_ret

172
173
174
175
176
177
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

178
        try:
179
            lora_request = self._maybe_get_adapters(
180
181
                request, supports_default_mm_loras=True
            )
182

183
            model_name = self.models.model_name(lora_request)
184

185
            tokenizer = await self.engine_client.get_tokenizer()
186

187
188
            tool_parser = self.tool_parser

189
            if isinstance(tokenizer, MistralTokenizer):
190
191
192
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
193
                maybe_serialize_tool_calls(request)
194
                truncate_tool_call_ids(request)
195
                validate_request_params(request)
196

197
198
199
200
201
202
            if (
                request.tool_choice == "auto"
                and not (self.enable_auto_tools and tool_parser is not None)
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
            ):
203
204
205
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
206
                    '"auto" tool choice requires '
207
208
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
209

210
211
212
213
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
214
215
216
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
217

218
219
            if not self.use_harmony:
                # Common case.
220
221
222
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
223
                    trust_request_chat_template=self.trust_request_chat_template,
224
225
226
                )
                if error_check_ret is not None:
                    return error_check_ret
227
228
229
230
231
232
233
234
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = await self._preprocess_chat(
                    request,
                    tokenizer,
                    request.messages,
235
                    chat_template=request.chat_template or self.chat_template,
236
                    chat_template_content_format=self.chat_template_content_format,
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = self._make_request_with_harmony(request)
252
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
253
            logger.exception("Error in preprocessing prompt inputs")
254
            return self.create_error_response(f"{e} {e.__cause__}")
255

256
257
258
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
259
260
261
262
263

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

264
265
266
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

267
        # Schedule the request and get the result generator.
268
        generators: list[AsyncGenerator[RequestOutput, None]] = []
269
        try:
270
            for i, engine_prompt in enumerate(engine_prompts):
271
                prompt_text, _, _ = self._get_prompt_components(request_prompts[i])
272
273
274
275
276
277
278
279

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
280
281
                    default_sampling_params=self.default_sampling_params,
                )
282

283
                sampling_params: SamplingParams | BeamSearchParams
284
285
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
286
287
                        max_tokens, self.default_sampling_params
                    )
288
289
                else:
                    sampling_params = request.to_sampling_params(
290
291
292
293
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
294

295
296
297
298
299
300
                self._log_inputs(
                    request_id,
                    request_prompts[i],
                    params=sampling_params,
                    lora_request=lora_request,
                )
301

302
303
304
305
306
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
307
308

                if isinstance(sampling_params, BeamSearchParams):
309
                    generator = self.beam_search(
310
311
312
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
313
                        lora_request=lora_request,
314
315
                    )
                else:
316
317
318
319
320
321
322
323
                    engine_request, tokenization_kwargs = await self._process_inputs(
                        request_id,
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
                    )
324

325
                    generator = self.engine_client.generate(
326
                        engine_request,
327
328
329
330
331
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
332
333
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
334
                        data_parallel_rank=data_parallel_rank,
335
336
337
                    )

                generators.append(generator)
338
        except ValueError as e:
339
            # TODO: Use a vllm-specific Validation Error
340
341
            return self.create_error_response(str(e))

342
        assert len(generators) == 1
343
        (result_generator,) = generators
344

345
346
347
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
348
349
350
351
352
353
354
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
355
            )
356

357
358
        try:
            return await self.chat_completion_full_generator(
359
360
361
362
363
364
365
366
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
367
368
369
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
370
371
372
373

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
374
        return request.messages[-1]["role"]
375

376
    @staticmethod
377
    def _bracket_level(s: str, opening="{", closing="}") -> int:
378
379
380
381
382
383
384
385
386
387
388
389
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
390
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
391
392
393
394
395
396
397
398
399
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
400
            if c == "{":
401
402
                bracket_level += 1
                passed_zero = bracket_level == 0
403
            elif c == "}":
404
405
406
407
408
409
410
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
411
                if c == ",":
412
413
414
415
416
417
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
418
        current_text: str | None,
419
420
        delta_text: str,
        function_name_returned: bool,
421
422
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
423
424
425
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
426
427
428
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
429
            logger.debug("not enough tokens to parse into JSON yet")
430
431
432
433
434
435
436
437
438
439
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
440
441
                delta_text, previous_text
            )
442
443
444
445
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
446
447
448
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
449
450
451
452
453
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
454
455
456
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
457
458
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
459
460
                        arguments, previous_text
                    )
461
462
463
464

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
465
                    if finishes_previous_tool and "parameters" not in current_tool_call:
466
467
468
                        current_tool_call = obj[-2]

                    function_name_returned = True
469
470
471
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
472
473
474
475
476
477
478
479
480
481
482
483
484
485
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
486
487
488

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
489
490
                        delta_text, previous_text
                    )
491
492

                    if delta_text != "":
493
494
495
496
497
498
499
500
501
502
503
504
505
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
506
507
508
509
510
                    else:
                        delta_message = None

        return delta_message, function_name_returned

511
    async def chat_completion_stream_generator(
512
513
514
515
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
516
        model_name: str,
517
        conversation: list[ConversationMessage],
518
        tokenizer: AnyTokenizer,
519
        request_metadata: RequestResponseMetadata,
520
    ) -> AsyncGenerator[str, None]:
521
        created_time = int(time.time())
522
        chunk_object_type: Final = "chat.completion.chunk"
523
        first_iteration = True
524
525

        # Send response for each token for each request.n (index)
526
527
528
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
529
        num_prompt_tokens = 0
530
        num_cached_tokens = None
531
532
        if self.use_harmony:
            harmony_parsers = [
533
                get_streamable_parser_for_assistant() for _ in range(num_choices)
534
            ]
535
536
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
537
538
539
540
541
542
543
544
545

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
546
547
            and self._should_stream_with_auto_tool_parsing(request)
        )
548

549
        all_previous_token_ids: list[list[int]] | None
550
        function_name_returned = [False] * num_choices
551
        if self.tool_call_id_type == "kimi_k2":
552
553
554
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
555

556
557
558
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

559
560
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
561
        if tool_choice_auto or self.reasoning_parser:
562
563
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
564
565
566
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
567
        else:
568
            all_previous_token_ids = None
569

570
        try:
571
            if self.reasoning_parser:
572
573
574
575
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
                    chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                )
576
577
578
579
580
581
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
582
583
584
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
585
                tool_parsers: list[ToolParser | None] = [
586
587
588
589
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
590
        except Exception as e:
591
            logger.exception("Error in tool parser creation.")
592
593
594
595
596
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

597
        stream_options = request.stream_options
598
599
600
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
601

602
603
        try:
            async for res in result_generator:
604
605
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
606
607
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
608

609
610
611
612
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
613
                    num_cached_tokens = res.num_cached_tokens
614
615
                    # Send first response for each request.n (index) with
                    # the role
616
                    role = self.get_chat_request_role(request)
617
618
619

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
620
                    for i in range(num_choices):
621
622
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
623
624
625
626
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
627
                            logprobs=None,
628
629
                            finish_reason=None,
                        )
630
631

                        # return prompt_token_ids at the first chunk ever
632
633
634
635
636
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
637
                            model=model_name,
638
639
640
641
642
643
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
644

645
646
647
648
649
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
650
651
                                total_tokens=num_prompt_tokens,
                            )
652

653
654
655
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

656
657
                    # Send response to echo the input portion of the
                    # last message
658
                    if request.echo:
659
                        last_msg_content: str | list[dict[str, str]] = ""
660
661
662
663
664
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
665
                            last_msg_content = conversation[-1]["content"] or ""
666
667

                        if last_msg_content:
668
                            for i in range(num_choices):
669
670
671
672
673
674
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
675
676
677
678
679
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
680
681
                                    model=model_name,
                                )
682
683
684
685
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
686
687
                                        total_tokens=num_prompt_tokens,
                                    )
688

689
                                data = chunk.model_dump_json(exclude_unset=True)
690
691
692
693
694
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
695
                    tool_parser = tool_parsers[i]
696
697
698
699

                    if finish_reason_sent[i]:
                        continue

700
                    if request.logprobs and request.top_logprobs is not None:
701
                        assert output.logprobs is not None, "Did not output logprobs"
702
                        logprobs = self._create_chat_logprobs(
703
704
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
705
                            tokenizer=tokenizer,
706
                            num_output_top_logprobs=request.top_logprobs,
707
                            return_as_token_id=request.return_tokens_as_token_ids,
708
709
710
711
                        )
                    else:
                        logprobs = None

712
713
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
714
                        prev_recipient = harmony_parser.current_recipient
715
                        delta_text = ""
716
717
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
718
                            delta_text += harmony_parser.last_content_delta or ""
719
720
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
721
722
                    else:
                        delta_text = output.text
723

724
725
726
727
728
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
729
730
731
                        # Chunked prefill case, don't return empty chunks
                        continue

732
                    delta_message: DeltaMessage | None
733

734
                    # just update previous_texts and previous_token_ids
735
                    if tool_choice_auto or self.reasoning_parser:
736
737
738
739
740
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
741
742
                        # avoid the None + list error.
                        if previous_token_ids:
743
                            current_token_ids = previous_token_ids + as_list(
744
745
                                output.token_ids
                            )
746
                        else:
747
                            current_token_ids = as_list(output.token_ids)
748

749
                    if self.use_harmony:
750
                        if cur_channel == "final":
751
                            delta_message = DeltaMessage(content=delta_text)
752
753
754
                        elif cur_channel == "analysis":
                            if request.include_reasoning:
                                delta_message = DeltaMessage(
755
756
                                    reasoning_content=delta_text
                                )
757
758
                            else:
                                delta_message = None
759
760
761
762
763
                        elif (
                            cur_channel == "commentary"
                            and cur_recipient
                            and cur_recipient.startswith("functions.")
                        ):
764
765
766
                            # Count completed tool calls to determine index
                            base_index = 0
                            for msg in harmony_parser.messages:
767
768
769
770
771
                                if (
                                    msg.channel == "commentary"
                                    and msg.recipient
                                    and msg.recipient.startswith("functions.")
                                ):
772
773
774
                                    base_index += 1

                            if prev_recipient != cur_recipient:
775
776
777
778
779
780
781
782
783
784
785
786
787
788
                                tool_name = cur_recipient.split("functions.", 1)[1]
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            id=make_tool_call_id(),
                                            type="function",
                                            function=DeltaFunctionCall(
                                                name=tool_name,
                                                arguments="",
                                            ),
                                            index=base_index,
                                        )
                                    ]
                                )
789
                            elif delta_text:
790
791
792
793
794
795
796
797
798
799
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            index=base_index,
                                            function=DeltaFunctionCall(
                                                arguments=delta_text
                                            ),
                                        )
                                    ]
                                )
800
801
802
803
804
805
806
                            else:
                                delta_message = None

                            if delta_message is not None:
                                harmony_tools_streamed[i] = True
                        else:
                            delta_message = None
807
                    # handle streaming deltas for tools with named tool_choice
808
                    elif tool_choice_function_name:
809
810
811
812
813
814
815
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
816
817
                            assert reasoning_parser is not None
                            delta_message = (
818
                                reasoning_parser.extract_reasoning_content_streaming(
819
820
821
822
823
824
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
825
826
                                )
                            )
827
828
829
830
831
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Only keep 'content', remove 'reasoning_content'.
832
                            if reasoning_parser.is_reasoning_end(
833
834
835
836
837
838
839
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
840
                                reasoning_end_arr[i] = True
841
842
843
844
845
846
847
848
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
849
                            if self.reasoning_parser:
850
851
852
                                delta_text = previous_text + delta_text
                                current_text = ""

853
854
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
855
856
857
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
858
859
                            else:
                                delta_tool_call = DeltaToolCall(
860
                                    id=make_tool_call_id(),
861
862
863
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
864
865
866
867
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
868
869
                                function_name_returned[i] = True

870
871
872
873
874
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
875
                            tools_streamed[i] = True
876

877
878
879
880
881
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
882
883
884
885
886
887
888
889
890
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
                            and res.prompt_token_ids
                            and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        ):
                            reasoning_end_arr[i] = True
891

892
893
894
895
896
897
898
899
900
901
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
                                reasoning_parser.extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
902
                            )
903
904
905
906
907
908
909
910
911
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

912
                        else:
913
                            # either finished reasoning or no reasoning at all
914
                            content = current_text
915
916
917
918
919
920
921
922
923

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
924
                            )
925
926
927
928
929
930
931
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
932

933
934
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
935
                    elif tool_choice_auto and self.reasoning_parser:
936
937
938
939
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
940
                        output_token_ids = as_list(output.token_ids)
941
942
                        if not reasoning_end_arr[i]:
                            delta_message = (
943
                                reasoning_parser.extract_reasoning_content_streaming(
944
945
946
947
948
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
949
                                    output_token_ids,
950
951
                                )
                            )
952
953
954
955
956
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
957
958
959
960
961
962
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
963
                                reasoning_end_arr[i] = True
964
                                current_token_ids = output_token_ids
965
966
967
968
969
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
970
971
972
973
                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
974
                            if reasoning_parser.is_reasoning_end(output_token_ids):
975
                                reasoning_end_arr[i] = True
976
                                current_token_ids = (
977
                                    reasoning_parser.extract_content_ids(
978
979
980
                                        output_token_ids
                                    )
                                )
981
982
983
984
985
986
987
988
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
989
                            delta_token_ids = output_token_ids
990
991
992
993
994
995
996
997
998
999
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1000
                            delta_message = tool_parser.extract_tool_calls_streaming(
1001
1002
                                previous_text=previous_text,
                                current_text=current_text,
1003
                                delta_text=delta_text,
1004
1005
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1023
1024
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1025

1026
                    # when only reasoning
1027
                    elif self.reasoning_parser:
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
                        delta_message = (
                            reasoning_parser.extract_reasoning_content_streaming(
                                previous_text,
                                current_text,
                                delta_text,
                                previous_token_ids,
                                current_token_ids,
                                output.token_ids,
                            )
                        )
1038
                    # handle streaming just a content delta
1039
1040
1041
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1042
                    # update the previous values for the next iteration
1043
1044
1045
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1046
1047
1048
1049
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1050
1051
1052
1053
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1054

1055
                    # set the previous values for the next iteration
1056
                    previous_num_tokens[i] += len(output.token_ids)
1057
1058
1059
1060
1061
1062

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1063
1064
1065
1066
                        if output.finish_reason is None:
                            continue
                        else:
                            delta_message = DeltaMessage()
1067

1068
1069
1070
1071
1072
1073
1074
1075
1076
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1077
1078
                                if tc.function and tc.function.arguments
                            )
1079
1080
1081
1082
1083

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1084
                                output_token_ids=as_list(output.token_ids),
1085
1086
1087
1088
1089
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1090
1091
1092
1093
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1094
                            delta=delta_message,
1095
                            logprobs=logprobs,
1096
                            finish_reason=None,
1097
1098
1099
1100
1101
1102
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1103
1104

                    # if the model is finished generating
1105
                    else:
1106
1107
1108
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1109
                        # only happens if we are NOT using structured outputs
1110
                        auto_tools_called = False
1111
                        if tool_parser:
1112
1113
1114
1115
1116
1117
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1118
1119
1120
                        else:
                            index = 0

1121
1122
1123
1124
1125
1126
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1127
                            latest_delta_len = 0
1128
1129
                            if (
                                isinstance(
1130
                                    delta_message.tool_calls[0].function,
1131
1132
1133
1134
1135
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1136
                                latest_delta_len = len(
1137
1138
                                    delta_message.tool_calls[0].function.arguments
                                )
1139

1140
1141
1142
1143
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1144
1145
1146
1147
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1148

1149
                            # get what we've streamed so far for arguments
1150
                            # for the current tool
1151
1152
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1153
                                actual_call = actual_call[:-latest_delta_len]
1154
1155

                            # check to see if there's anything left to stream
1156
                            remaining_call = expected_call.replace(actual_call, "", 1)
1157
                            # set that as a delta message
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    DeltaToolCall(
                                        index=index,
                                        function=DeltaFunctionCall(
                                            arguments=remaining_call
                                        ).model_dump(exclude_none=True),
                                    )
                                ]
                            )
1168

1169
                        # Send the finish response for each request.n only once
1170
1171
1172
1173
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1174
1175
                        if (
                            auto_tools_called
1176
                            or (tools_streamed[i] and not tool_choice_function_name)
1177
1178
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1179
1180
                            finish_reason_ = "tool_calls"
                        else:
1181
1182
1183
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1184
1185
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1186
                            delta=delta_message,
1187
                            logprobs=logprobs,
1188
                            finish_reason=finish_reason_,
1189
                            stop_reason=output.stop_reason,
1190
1191
1192
1193
1194
1195
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1196

1197
                        finish_reason_sent[i] = True
1198

1199
1200
1201
1202
1203
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1204
1205
                        model=model_name,
                    )
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1216
                    data = chunk.model_dump_json(exclude_unset=True)
1217
1218
                    yield f"data: {data}\n\n"

1219
1220
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1221
1222
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1223
1224
1225
1226
1227
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1228
1229
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1230
1231
                        cached_tokens=num_cached_tokens
                    )
1232
1233
1234
1235
1236
1237
1238

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1239
1240
1241
1242
1243
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1244
                yield f"data: {final_usage_data}\n\n"
1245

1246
1247
1248
1249
1250
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1251
1252
1253
1254
1255
1256
1257
1258
1259
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1260
1261
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1262
1263
1264
1265
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1266
                        output_token_ids=None,  # Consider also logging all token IDs
1267
1268
1269
1270
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1271

1272
        except Exception as e:
1273
            # TODO: Use a vllm-specific Validation Error
1274
            logger.exception("Error in chat completion stream generator.")
1275
1276
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1277
1278
1279
1280
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1281
1282
1283
1284
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1285
        model_name: str,
1286
        conversation: list[ConversationMessage],
1287
        tokenizer: AnyTokenizer,
1288
        request_metadata: RequestResponseMetadata,
1289
    ) -> ErrorResponse | ChatCompletionResponse:
1290
        created_time = int(time.time())
1291
        final_res: RequestOutput | None = None
1292

1293
1294
1295
1296
1297
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1298
1299
1300
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1301

1302
1303
        assert final_res is not None

1304
        choices: list[ChatCompletionResponseChoice] = []
1305
        if self.tool_call_id_type == "kimi_k2":
1306
1307
1308
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1309

1310
1311
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1312
            token_ids = output.token_ids
1313
            out_logprobs = output.logprobs
1314
            tool_call_info = None
1315

1316
1317
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1318
                logprobs = self._create_chat_logprobs(
1319
                    token_ids=token_ids,
1320
                    top_logprobs=out_logprobs,
1321
                    num_output_top_logprobs=request.top_logprobs,
1322
                    tokenizer=tokenizer,
1323
                    return_as_token_id=request.return_tokens_as_token_ids,
1324
1325
1326
                )
            else:
                logprobs = None
1327
1328

            if self.use_harmony:
1329
1330
1331
1332
                reasoning_content, content, _ = parse_chat_output(token_ids)
                if not request.include_reasoning:
                    reasoning_content = None

1333
1334
1335
1336
1337
1338
1339
1340
                if self.tool_parser is not None:
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1341
                    content = tool_call_info.content
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                    )
1354
1355
1356
1357
1358

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1359
1360
1361
1362
1363
1364
1365
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1366
1367
1368
1369
                    stop_reason=output.stop_reason,
                )
                choices.append(choice_data)
                continue
1370

1371
            if self.reasoning_parser:
1372
                try:
1373
1374
1375
1376
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
                        chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                    )
1377
1378
1379
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1380
1381
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1382
1383
1384
                reasoning_content, content = reasoning_parser.extract_reasoning_content(
                    output.text, request=request
                )
1385
1386
                if not request.include_reasoning:
                    reasoning_content = None
1387
1388
1389
            else:
                reasoning_content = None
                content = output.text
1390

1391
            auto_tools_called = False
1392
1393
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1404
1405
1406
1407
1408
1409
1410
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1411
1412

            # if the request uses tools and specified a tool choice
1413
1414
1415
1416
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1417
                assert tool_calls is not None and len(tool_calls) > 0
1418
1419
                message = ChatMessage(
                    role=role,
1420
                    reasoning_content=reasoning_content,
1421
                    content="",
1422
                    tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1423
                )
1424

1425
            elif request.tool_choice and request.tool_choice == "required":
1426
1427
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1428
                for tool_call in tool_calls:
1429
1430
1431
1432
1433
1434
1435
1436
                    tool_call_class_items.append(
                        tool_call_class(
                            id=make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
                                idx=history_tool_call_cnt,
                            ),
                            function=tool_call,
1437
1438
                        )
                    )
1439
                    history_tool_call_cnt += 1
1440
1441
1442
                message = ChatMessage(
                    role=role,
                    content="",
1443
                    tool_calls=tool_call_class_items,
1444
1445
                    reasoning_content=reasoning_content,
                )
1446

1447
1448
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1449
            elif not request.tool_choice or request.tool_choice == "none":
1450
1451
1452
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1453
1454

            # handle when there are tools and tool choice is auto
1455
1456
1457
1458
1459
1460
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1461
1462
1463
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1464
1465
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1466
1467
1468
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
1469
1470
1471
1472
1473
1474
1475
1476
                        content=content,
                        tool_calls=[
                            ToolCall(
                                function=tc,
                                type="function",
                            )
                            for tc in tool_calls
                        ],
1477
                    )
1478
1479
1480
1481

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1482
1483
1484
1485
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1486
1487
                    if content and len(content) > 0:
                        ret_content = content
1488
1489
1490
1491
1492
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=ret_content,
                    )
1493
1494
1495
1496
1497
1498

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1499
1500
1501
1502
1503
                    "completion."
                )
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1504
1505
1506
1507
1508
1509
1510
1511
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1512

1513
1514
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1515
                message=message,
1516
                logprobs=logprobs,
1517
1518
1519
1520
1521
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1522
                stop_reason=output.stop_reason,
1523
1524
1525
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1526
            )
1527

1528
1529
            choices.append(choice_data)

1530
        if request.echo:
1531
            last_msg_content: str | list[dict[str, str]] = ""
1532
1533
1534
1535
1536
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1537
                last_msg_content = conversation[-1]["content"] or ""
1538
            if isinstance(last_msg_content, list):
1539
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1540
1541

            for choice in choices:
1542
                full_message = last_msg_content + (choice.message.content or "")
1543
1544
                choice.message.content = full_message

1545
        assert final_res.prompt_token_ids is not None
1546
        num_prompt_tokens = len(final_res.prompt_token_ids)
1547
1548
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1549
        num_generated_tokens = sum(
1550
1551
1552
1553
1554
1555
1556
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1557
1558
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1559
1560
                cached_tokens=final_res.num_cached_tokens
            )
1561
1562
1563

        request_metadata.final_usage_info = usage

1564
1565
1566
1567
1568
1569
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1570
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1571
1572
1573
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1574
            kv_transfer_params=final_res.kv_transfer_params,
1575
1576
        )

1577
1578
1579
1580
1581
1582
1583
1584
1585
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1586
1587
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1588
1589
                            tc.function, "arguments"
                        ):
1590
                            tool_call_descriptions.append(
1591
1592
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1593
1594
1595
1596
1597
1598
1599
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1600
                        output_token_ids = final_res.outputs[choice.index].token_ids
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1611
        return response
1612
1613

    def _get_top_logprobs(
1614
1615
        self,
        logprobs: dict[int, Logprob],
1616
        top_logprobs: int | None,
1617
1618
1619
        tokenizer: AnyTokenizer,
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1620
        return [
1621
            ChatCompletionLogProb(
1622
1623
1624
1625
1626
1627
1628
1629
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1630
1631
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1632
1633
            )
            for i, p in enumerate(logprobs.items())
1634
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1635
1636
1637
1638
1639
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1640
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1641
        tokenizer: AnyTokenizer,
1642
1643
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1644
1645
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1646
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1647

1648
1649
1650
1651
1652
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1653
1654
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1655
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1656
                if should_return_as_token_id:
1657
                    token = f"token_id:{token_id}"
1658
1659
                else:
                    token = tokenizer.decode(token_id)
1660

1661
1662
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1663
                        token=token,
1664
                        bytes=list(token.encode("utf-8", errors="replace")),
1665
1666
                    )
                )
1667
            else:
1668
1669
1670
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1671
1672
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1673
                        token=self._get_decoded_token(
1674
1675
1676
                            step_token,
                            token_id,
                            tokenizer,
1677
                            should_return_as_token_id,
1678
1679
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1680
1681
1682
1683
1684
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1685
                        top_logprobs=self._get_top_logprobs(
1686
1687
1688
1689
1690
1691
1692
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1693
1694

        return ChatCompletionLogProbs(content=logprobs_content)
1695

1696
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1697
1698
1699
1700
1701
1702
1703
1704
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1705
1706
1707
1708
1709
1710
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1711
1712
1713

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1714
        delta_message: DeltaMessage | None,
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1726
            output.finish_reason is not None
1727
1728
1729
1730
1731
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1732
1733
1734
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
    ):
        messages: list[OpenAIMessage] = []

        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1750
            python_description=None,
1751
1752
            with_custom_tools=request.tools is not None,
        )
1753
1754
1755
        messages.append(sys_msg)

        # Add developer message.
1756
        dev_msg = get_developer_message(tools=request.tools)
1757
1758
1759
1760
        messages.append(dev_msg)

        # Add user message.
        for chat_msg in request.messages:
1761
            messages.extend(parse_input_to_harmony_message(chat_msg))
1762
1763
1764
1765

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1766
1767
1768
1769
1770

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1771
        return messages, [prompt_token_ids], [engine_prompt]