serving_chat.py 78.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
22
23
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
24
25
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.parser.harmony_utils import (
26
27
28
29
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
30
    parse_chat_inputs_to_harmony_messages,
31
32
33
    parse_chat_output,
    render_for_completion,
)
34
from vllm.entrypoints.openai.protocol import (
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
54
55
56
57
58
from vllm.entrypoints.openai.serving_engine import (
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
59
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
60
from vllm.entrypoints.openai.tool_parsers import ToolParser
61
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
62
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
63
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
64
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
65
from vllm.logger import init_logger
66
from vllm.logprobs import Logprob
67
from vllm.outputs import CompletionOutput, RequestOutput
68
from vllm.sampling_params import BeamSearchParams, SamplingParams
69
70
71
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
    MistralTokenizer,
72
73
74
75
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
76
from vllm.utils.collection_utils import as_list
77
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
78
79
80
81
82

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
83
84
85
    def __init__(
        self,
        engine_client: EngineClient,
86
        models: OpenAIServingModels,
87
88
        response_role: str,
        *,
89
90
        request_logger: RequestLogger | None,
        chat_template: str | None,
91
        chat_template_content_format: ChatTemplateContentFormatOption,
92
        trust_request_chat_template: bool = False,
93
        return_tokens_as_token_ids: bool = False,
94
        reasoning_parser: str = "",
95
        enable_auto_tools: bool = False,
96
        exclude_tools_when_tool_choice_none: bool = False,
97
        tool_parser: str | None = None,
98
        enable_prompt_tokens_details: bool = False,
99
        enable_force_include_usage: bool = False,
100
        enable_log_outputs: bool = False,
101
        log_error_stack: bool = False,
102
    ) -> None:
103
104
105
106
107
108
109
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
110

111
        self.response_role = response_role
112
113
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
114
        self.trust_request_chat_template = trust_request_chat_template
115
        self.enable_log_outputs = enable_log_outputs
116

117
118
119
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

120
121
122
123
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
124
125
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
126
127
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
128
129
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
130

131
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
132
        self.enable_force_include_usage = enable_force_include_usage
133
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
134
        if self.default_sampling_params:
135
136
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
137
138
139
140
141
142
143
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
144
        else:
145
            self.tool_call_id_type = "random"
146

147
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
148
149
150
151
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
152
153
                get_stop_tokens_for_assistant_actions()
            )
154
155
156
157
158
159
160
161
162
163
164

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

165
    async def create_chat_completion(
166
167
        self,
        request: ChatCompletionRequest,
168
169
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
170
171
        """
        Chat Completion API similar to OpenAI's API.
172

173
174
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
175
        Chat Completion API.
176
177
178
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
179
            logger.error("Error with model %s", error_check_ret)
180
181
            return error_check_ret

182
183
184
185
186
187
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

188
        try:
189
            lora_request = self._maybe_get_adapters(
190
191
                request, supports_default_mm_loras=True
            )
192

193
            model_name = self.models.model_name(lora_request)
194

195
            tokenizer = await self.engine_client.get_tokenizer()
196

197
198
            tool_parser = self.tool_parser

199
            if isinstance(tokenizer, MistralTokenizer):
200
201
202
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
203
                maybe_serialize_tool_calls(request)
204
                truncate_tool_call_ids(request)
205
                validate_request_params(request)
206

207
208
209
210
211
212
            if (
                request.tool_choice == "auto"
                and not (self.enable_auto_tools and tool_parser is not None)
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
            ):
213
214
215
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
216
                    '"auto" tool choice requires '
217
218
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
219

220
221
222
223
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
224
225
226
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
227

228
229
            if not self.use_harmony:
                # Common case.
230
231
232
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
233
                    trust_request_chat_template=self.trust_request_chat_template,
234
235
236
                )
                if error_check_ret is not None:
                    return error_check_ret
237
238
239
240
241
242
243
244
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = await self._preprocess_chat(
                    request,
                    tokenizer,
                    request.messages,
245
                    chat_template=request.chat_template or self.chat_template,
246
                    chat_template_content_format=self.chat_template_content_format,
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = self._make_request_with_harmony(request)
262
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
263
            logger.exception("Error in preprocessing prompt inputs")
264
            return self.create_error_response(f"{e} {e.__cause__}")
265

266
267
268
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
269
270
271
272
273

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

274
275
276
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

277
        # Schedule the request and get the result generator.
278
        generators: list[AsyncGenerator[RequestOutput, None]] = []
279
        try:
280
            for i, engine_prompt in enumerate(engine_prompts):
281
                prompt_text, _, _ = self._get_prompt_components(request_prompts[i])
282
283
284
285
286
                # If we are creating sub requests for multiple prompts, ensure that they
                # have unique request ids.
                sub_request_id = (
                    request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                )
287
288
289
290
291
292
293
294

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
295
296
                    default_sampling_params=self.default_sampling_params,
                )
297

298
                sampling_params: SamplingParams | BeamSearchParams
299
300
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
301
302
                        max_tokens, self.default_sampling_params
                    )
303
304
                else:
                    sampling_params = request.to_sampling_params(
305
306
307
308
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
309
310
311
312
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
313

314
                self._log_inputs(
315
                    sub_request_id,
316
317
318
319
                    request_prompts[i],
                    params=sampling_params,
                    lora_request=lora_request,
                )
320

321
322
323
324
325
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
326
327

                if isinstance(sampling_params, BeamSearchParams):
328
                    generator = self.beam_search(
329
                        prompt=engine_prompt,
330
                        request_id=sub_request_id,
331
                        params=sampling_params,
332
                        lora_request=lora_request,
333
                        trace_headers=trace_headers,
334
335
                    )
                else:
336
                    engine_request, tokenization_kwargs = await self._process_inputs(
337
                        sub_request_id,
338
339
340
341
342
343
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
                    )
344

345
                    generator = self.engine_client.generate(
346
                        engine_request,
347
                        sampling_params,
348
                        sub_request_id,
349
350
351
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
352
353
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
354
                        data_parallel_rank=data_parallel_rank,
355
356
357
                    )

                generators.append(generator)
358
        except ValueError as e:
359
            # TODO: Use a vllm-specific Validation Error
360
361
            return self.create_error_response(str(e))

362
        assert len(generators) == 1
363
        (result_generator,) = generators
364

365
366
367
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
368
369
370
371
372
373
374
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
375
            )
376

377
378
        try:
            return await self.chat_completion_full_generator(
379
380
381
382
383
384
385
386
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
387
388
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
389
390
391
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
392
393
394
395

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
396
        return request.messages[-1]["role"]
397

398
    @staticmethod
399
    def _bracket_level(s: str, opening="{", closing="}") -> int:
400
401
402
403
404
405
406
407
408
409
410
411
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
412
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
413
414
415
416
417
418
419
420
421
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
422
            if c == "{":
423
424
                bracket_level += 1
                passed_zero = bracket_level == 0
425
            elif c == "}":
426
427
428
429
430
431
432
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
433
                if c == ",":
434
435
436
437
438
439
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
440
        current_text: str | None,
441
442
        delta_text: str,
        function_name_returned: bool,
443
444
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
445
446
447
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
448
449
450
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
451
            logger.debug("not enough tokens to parse into JSON yet")
452
453
454
455
456
457
458
459
460
461
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
462
463
                delta_text, previous_text
            )
464
465
466
467
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
468
469
470
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
471
472
473
474
475
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
476
477
478
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
479
480
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
481
482
                        arguments, previous_text
                    )
483
484
485
486

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
487
                    if finishes_previous_tool and "parameters" not in current_tool_call:
488
489
490
                        current_tool_call = obj[-2]

                    function_name_returned = True
491
492
493
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
494
495
496
497
498
499
500
501
502
503
504
505
506
507
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
508
509
510

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
511
512
                        delta_text, previous_text
                    )
513
514

                    if delta_text != "":
515
516
517
518
519
520
521
522
523
524
525
526
527
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
528
529
530
531
532
                    else:
                        delta_message = None

        return delta_message, function_name_returned

533
    async def chat_completion_stream_generator(
534
535
536
537
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
538
        model_name: str,
539
        conversation: list[ConversationMessage],
540
        tokenizer: TokenizerLike,
541
        request_metadata: RequestResponseMetadata,
542
    ) -> AsyncGenerator[str, None]:
543
        created_time = int(time.time())
544
        chunk_object_type: Final = "chat.completion.chunk"
545
        first_iteration = True
546
547

        # Send response for each token for each request.n (index)
548
549
550
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
551
        num_prompt_tokens = 0
552
        num_cached_tokens = None
553
554
        if self.use_harmony:
            harmony_parsers = [
555
                get_streamable_parser_for_assistant() for _ in range(num_choices)
556
            ]
557
558
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
559
560
561
562
563
564
565
566
567

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
568
569
            and self._should_stream_with_auto_tool_parsing(request)
        )
570

571
        all_previous_token_ids: list[list[int]] | None
572
        function_name_returned = [False] * num_choices
573
        if self.tool_call_id_type == "kimi_k2":
574
575
576
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
577

578
579
580
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

581
582
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
583
        if tool_choice_auto or self.reasoning_parser:
584
585
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
586
587
588
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
589
        else:
590
            all_previous_token_ids = None
591

592
        try:
593
            if self.reasoning_parser:
594
595
596
597
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
                    chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                )
598
599
600
601
602
603
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
604
605
606
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
607
                tool_parsers: list[ToolParser | None] = [
608
609
610
611
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
612
        except Exception as e:
613
            logger.exception("Error in tool parser creation.")
614
615
616
617
618
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

619
        stream_options = request.stream_options
620
621
622
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
623

624
625
        try:
            async for res in result_generator:
626
627
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
628
629
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
630

631
632
633
634
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
635
                    num_cached_tokens = res.num_cached_tokens
636
637
                    # Send first response for each request.n (index) with
                    # the role
638
                    role = self.get_chat_request_role(request)
639
640
641

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
642
                    for i in range(num_choices):
643
644
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
645
646
647
648
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
649
                            logprobs=None,
650
651
                            finish_reason=None,
                        )
652
653

                        # return prompt_token_ids at the first chunk ever
654
655
656
657
658
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
659
                            model=model_name,
660
661
662
663
664
665
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
666

667
668
669
670
671
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
672
673
                                total_tokens=num_prompt_tokens,
                            )
674

675
676
677
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

678
679
                    # Send response to echo the input portion of the
                    # last message
680
                    if request.echo:
681
                        last_msg_content: str | list[dict[str, str]] = ""
682
683
684
685
686
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
687
                            last_msg_content = conversation[-1]["content"] or ""
688
689

                        if last_msg_content:
690
                            for i in range(num_choices):
691
692
693
694
695
696
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
697
698
699
700
701
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
702
703
                                    model=model_name,
                                )
704
705
706
707
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
708
709
                                        total_tokens=num_prompt_tokens,
                                    )
710

711
                                data = chunk.model_dump_json(exclude_unset=True)
712
713
714
715
716
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
717
                    tool_parser = tool_parsers[i]
718
719
720
721

                    if finish_reason_sent[i]:
                        continue

722
                    if request.logprobs and request.top_logprobs is not None:
723
                        assert output.logprobs is not None, "Did not output logprobs"
724
                        logprobs = self._create_chat_logprobs(
725
726
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
727
                            tokenizer=tokenizer,
728
                            num_output_top_logprobs=request.top_logprobs,
729
                            return_as_token_id=request.return_tokens_as_token_ids,
730
731
732
733
                        )
                    else:
                        logprobs = None

734
735
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
736
                        prev_recipient = harmony_parser.current_recipient
737
                        delta_text = ""
738
739
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
740
                            delta_text += harmony_parser.last_content_delta or ""
741
742
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
743
744
                    else:
                        delta_text = output.text
745

746
747
748
749
750
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
751
752
753
                        # Chunked prefill case, don't return empty chunks
                        continue

754
                    delta_message: DeltaMessage | None
755

756
                    # just update previous_texts and previous_token_ids
757
                    if tool_choice_auto or self.reasoning_parser:
758
759
760
761
762
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
763
764
                        # avoid the None + list error.
                        if previous_token_ids:
765
                            current_token_ids = previous_token_ids + as_list(
766
767
                                output.token_ids
                            )
768
                        else:
769
                            current_token_ids = as_list(output.token_ids)
770

771
                    if self.use_harmony:
772
                        if cur_channel == "final":
773
                            delta_message = DeltaMessage(content=delta_text)
774
775
                        elif cur_channel == "analysis":
                            if request.include_reasoning:
776
                                delta_message = DeltaMessage(reasoning=delta_text)
777
778
                            else:
                                delta_message = None
779
780
781
782
783
                        elif (
                            cur_channel == "commentary"
                            and cur_recipient
                            and cur_recipient.startswith("functions.")
                        ):
784
785
786
                            # Count completed tool calls to determine index
                            base_index = 0
                            for msg in harmony_parser.messages:
787
788
789
790
791
                                if (
                                    msg.channel == "commentary"
                                    and msg.recipient
                                    and msg.recipient.startswith("functions.")
                                ):
792
793
794
                                    base_index += 1

                            if prev_recipient != cur_recipient:
795
796
797
798
799
800
801
802
803
804
805
806
807
808
                                tool_name = cur_recipient.split("functions.", 1)[1]
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            id=make_tool_call_id(),
                                            type="function",
                                            function=DeltaFunctionCall(
                                                name=tool_name,
                                                arguments="",
                                            ),
                                            index=base_index,
                                        )
                                    ]
                                )
809
                            elif delta_text:
810
811
812
813
814
815
816
817
818
819
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            index=base_index,
                                            function=DeltaFunctionCall(
                                                arguments=delta_text
                                            ),
                                        )
                                    ]
                                )
820
821
822
823
824
                            else:
                                delta_message = None

                            if delta_message is not None:
                                harmony_tools_streamed[i] = True
825
826
827
                        elif cur_channel == "commentary":
                            # Tool call preambles meant to be shown to the user
                            delta_message = DeltaMessage(content=delta_text)
828
829
                        else:
                            delta_message = None
830
                    # handle streaming deltas for tools with named tool_choice
831
                    elif tool_choice_function_name:
832
833
834
835
836
837
838
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
839
840
                            assert reasoning_parser is not None
                            delta_message = (
841
                                reasoning_parser.extract_reasoning_streaming(
842
843
844
845
846
847
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
848
849
                                )
                            )
850
851
852
853
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
854
                            # Only keep 'content', remove 'reasoning'.
855
                            if reasoning_parser.is_reasoning_end(
856
857
858
859
860
861
862
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
863
                                reasoning_end_arr[i] = True
864
865
866
867
868
869
870
871
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
872
                            if self.reasoning_parser:
873
874
875
                                delta_text = previous_text + delta_text
                                current_text = ""

876
877
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
878
879
880
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
881
882
                            else:
                                delta_tool_call = DeltaToolCall(
883
                                    id=make_tool_call_id(),
884
885
886
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
887
888
889
890
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
891
892
                                function_name_returned[i] = True

893
894
895
896
897
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
898
                            tools_streamed[i] = True
899

900
901
902
903
904
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
905
906
907
908
909
910
911
912
913
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
                            and res.prompt_token_ids
                            and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        ):
                            reasoning_end_arr[i] = True
914

915
916
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
917
                                reasoning_parser.extract_reasoning_streaming(
918
919
920
921
922
923
924
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
925
                            )
926
927
928
929
930
931
932
933
934
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

935
                        else:
936
                            # either finished reasoning or no reasoning at all
937
                            content = current_text
938
939
940
941
942
943
944
945
946

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
947
                            )
948
949
950
951
952
953
954
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
955

956
957
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
958
                    elif tool_choice_auto and self.reasoning_parser:
959
960
961
962
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
963
                        output_token_ids = as_list(output.token_ids)
964
965
                        if not reasoning_end_arr[i]:
                            delta_message = (
966
                                reasoning_parser.extract_reasoning_streaming(
967
968
969
970
971
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
972
                                    output_token_ids,
973
974
                                )
                            )
975
976
977
978
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Remove the text and token ids related
979
                            # to 'reasoning'.
980
981
982
983
984
985
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
986
                                reasoning_end_arr[i] = True
987
                                current_token_ids = output_token_ids
988
989
990
991
992
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
993
994
995
                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
996
                            # to 'reasoning'.
997
                            if reasoning_parser.is_reasoning_end(output_token_ids):
998
                                reasoning_end_arr[i] = True
999
                                current_token_ids = (
1000
                                    reasoning_parser.extract_content_ids(
1001
1002
1003
                                        output_token_ids
                                    )
                                )
1004
1005
1006
1007
1008
1009
1010
1011
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
1012
                            delta_token_ids = output_token_ids
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1023
                            delta_message = tool_parser.extract_tool_calls_streaming(
1024
1025
                                previous_text=previous_text,
                                current_text=current_text,
1026
                                delta_text=delta_text,
1027
1028
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1046
1047
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1048

1049
                    # when only reasoning
1050
                    elif self.reasoning_parser:
1051
1052
1053
1054
1055
1056
1057
                        delta_message = reasoning_parser.extract_reasoning_streaming(
                            previous_text,
                            current_text,
                            delta_text,
                            previous_token_ids,
                            current_token_ids,
                            output.token_ids,
1058
                        )
1059
                    # handle streaming just a content delta
1060
1061
1062
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1063
                    # update the previous values for the next iteration
1064
1065
1066
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1067
1068
1069
1070
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1071
1072
1073
1074
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1075

1076
                    # set the previous values for the next iteration
1077
                    previous_num_tokens[i] += len(output.token_ids)
1078
1079
1080
1081
1082
1083

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1084
1085
1086
1087
1088
1089
1090
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1091
                            continue
1092
                        delta_message = DeltaMessage()
1093

1094
1095
1096
1097
1098
1099
1100
1101
1102
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1103
1104
                                if tc.function and tc.function.arguments
                            )
1105
1106
1107
1108
1109

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1110
                                output_token_ids=as_list(output.token_ids),
1111
1112
1113
1114
1115
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1116
1117
1118
1119
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1120
                            delta=delta_message,
1121
                            logprobs=logprobs,
1122
                            finish_reason=None,
1123
1124
1125
1126
1127
1128
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1129
1130

                    # if the model is finished generating
1131
                    else:
1132
1133
1134
1135
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1136
1137
1138
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1139
                        # only happens if we are NOT using structured outputs
1140
                        auto_tools_called = False
1141
                        if tool_parser:
1142
1143
1144
1145
1146
1147
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1148
1149
1150
                        else:
                            index = 0

1151
1152
1153
1154
1155
1156
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1157
                            latest_delta_len = 0
1158
1159
                            if (
                                isinstance(
1160
                                    delta_message.tool_calls[0].function,
1161
1162
1163
1164
1165
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1166
                                latest_delta_len = len(
1167
1168
                                    delta_message.tool_calls[0].function.arguments
                                )
1169

1170
1171
1172
1173
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1174
1175
1176
1177
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1178

1179
                            # get what we've streamed so far for arguments
1180
                            # for the current tool
1181
1182
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1183
                                actual_call = actual_call[:-latest_delta_len]
1184
1185

                            # check to see if there's anything left to stream
1186
                            remaining_call = expected_call.replace(actual_call, "", 1)
1187
                            # set that as a delta message
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    DeltaToolCall(
                                        index=index,
                                        function=DeltaFunctionCall(
                                            arguments=remaining_call
                                        ).model_dump(exclude_none=True),
                                    )
                                ]
                            )
1198

1199
                        # Send the finish response for each request.n only once
1200
1201
1202
1203
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1204
1205
                        if (
                            auto_tools_called
1206
                            or (tools_streamed[i] and not tool_choice_function_name)
1207
1208
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1209
1210
                            finish_reason_ = "tool_calls"
                        else:
1211
1212
1213
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1214
1215
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1216
                            delta=delta_message,
1217
                            logprobs=logprobs,
1218
                            finish_reason=finish_reason_,
1219
                            stop_reason=output.stop_reason,
1220
1221
1222
1223
1224
1225
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1226

1227
                        finish_reason_sent[i] = True
1228

1229
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1230
1231
1232
1233
1234
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1235
1236
                        model=model_name,
                    )
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1247
                    data = chunk.model_dump_json(exclude_unset=True)
1248
1249
                    yield f"data: {data}\n\n"

1250
1251
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1252
1253
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1254
1255
1256
1257
1258
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1259
1260
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1261
1262
                        cached_tokens=num_cached_tokens
                    )
1263
1264
1265
1266
1267
1268
1269

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1270
1271
1272
1273
1274
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1275
                yield f"data: {final_usage_data}\n\n"
1276

1277
1278
1279
1280
1281
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1282
1283
1284
1285
1286
1287
1288
1289
1290
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1291
1292
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1293
1294
1295
1296
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1297
                        output_token_ids=None,  # Consider also logging all token IDs
1298
1299
1300
1301
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1302

1303
1304
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1305
        except Exception as e:
1306
            # TODO: Use a vllm-specific Validation Error
1307
            logger.exception("Error in chat completion stream generator.")
1308
1309
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1310
1311
1312
1313
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1314
1315
1316
1317
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1318
        model_name: str,
1319
        conversation: list[ConversationMessage],
1320
        tokenizer: TokenizerLike,
1321
        request_metadata: RequestResponseMetadata,
1322
    ) -> ErrorResponse | ChatCompletionResponse:
1323
        created_time = int(time.time())
1324
        final_res: RequestOutput | None = None
1325

1326
1327
1328
1329
1330
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1331
1332
1333
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1334

1335
1336
        assert final_res is not None

1337
        choices: list[ChatCompletionResponseChoice] = []
1338
        if self.tool_call_id_type == "kimi_k2":
1339
1340
1341
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1342

1343
1344
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1345
1346
1347
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1348
            token_ids = output.token_ids
1349
            out_logprobs = output.logprobs
1350
            tool_call_info = None
1351

1352
1353
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1354
                logprobs = self._create_chat_logprobs(
1355
                    token_ids=token_ids,
1356
                    top_logprobs=out_logprobs,
1357
                    num_output_top_logprobs=request.top_logprobs,
1358
                    tokenizer=tokenizer,
1359
                    return_as_token_id=request.return_tokens_as_token_ids,
1360
1361
1362
                )
            else:
                logprobs = None
1363
1364

            if self.use_harmony:
1365
                reasoning, content, _ = parse_chat_output(token_ids)
1366
                if not request.include_reasoning:
1367
                    reasoning = None
1368

1369
1370
1371
1372
1373
1374
1375
1376
                if self.tool_parser is not None:
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1377
                    content = tool_call_info.content
1378
1379
                    message = ChatMessage(
                        role=role,
1380
                        reasoning=reasoning,
1381
1382
1383
1384
1385
1386
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1387
                        reasoning=reasoning,
1388
1389
                        content=content,
                    )
1390
1391
1392
1393
1394

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1395
1396
1397
1398
1399
1400
1401
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1402
                    stop_reason=output.stop_reason,
1403
1404
1405
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1406
1407
1408
                )
                choices.append(choice_data)
                continue
1409

1410
            if self.reasoning_parser:
1411
                try:
1412
1413
1414
1415
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
                        chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                    )
1416
1417
1418
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1419
1420
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1421
                reasoning, content = reasoning_parser.extract_reasoning(
1422
1423
                    output.text, request=request
                )
1424
                if not request.include_reasoning:
1425
                    reasoning = None
1426
            else:
1427
                reasoning = None
1428
                content = output.text
1429

1430
            auto_tools_called = False
1431
1432
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1443
1444
1445
1446
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1447
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1448
1449

            # if the request uses tools and specified a tool choice
1450
1451
1452
1453
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1454
                assert tool_calls is not None and len(tool_calls) > 0
1455
1456
                message = ChatMessage(
                    role=role,
1457
                    reasoning=reasoning,
1458
                    content="",
1459
                    tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1460
                )
1461

1462
            elif request.tool_choice and request.tool_choice == "required":
1463
1464
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1465
                for tool_call in tool_calls:
1466
1467
1468
1469
1470
1471
1472
1473
                    tool_call_class_items.append(
                        tool_call_class(
                            id=make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
                                idx=history_tool_call_cnt,
                            ),
                            function=tool_call,
1474
1475
                        )
                    )
1476
                    history_tool_call_cnt += 1
1477
1478
1479
                message = ChatMessage(
                    role=role,
                    content="",
1480
                    tool_calls=tool_call_class_items,
1481
                    reasoning=reasoning,
1482
                )
1483

1484
1485
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1486
            elif not request.tool_choice or request.tool_choice == "none":
1487
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1488
1489

            # handle when there are tools and tool choice is auto
1490
1491
1492
1493
1494
1495
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1496
1497
1498
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1499
1500
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1501
1502
                    message = ChatMessage(
                        role=role,
1503
                        reasoning=reasoning,
1504
1505
1506
1507
1508
1509
1510
1511
                        content=content,
                        tool_calls=[
                            ToolCall(
                                function=tc,
                                type="function",
                            )
                            for tc in tool_calls
                        ],
1512
                    )
1513
1514
1515
1516

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1517
1518
1519
1520
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1521
1522
                    if content and len(content) > 0:
                        ret_content = content
1523
1524
                    message = ChatMessage(
                        role=role,
1525
                        reasoning=reasoning,
1526
1527
                        content=ret_content,
                    )
1528
1529
1530
1531
1532
1533

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1534
1535
                    "completion."
                )
1536
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1537
1538
1539
1540
1541
1542
1543
1544
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1545

1546
1547
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1548
                message=message,
1549
                logprobs=logprobs,
1550
1551
1552
1553
1554
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1555
                stop_reason=output.stop_reason,
1556
1557
1558
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1559
            )
1560
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1561

1562
1563
            choices.append(choice_data)

1564
        if request.echo:
1565
            last_msg_content: str | list[dict[str, str]] = ""
1566
1567
1568
1569
1570
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1571
                last_msg_content = conversation[-1]["content"] or ""
1572
            if isinstance(last_msg_content, list):
1573
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1574
1575

            for choice in choices:
1576
                full_message = last_msg_content + (choice.message.content or "")
1577
1578
                choice.message.content = full_message

1579
        assert final_res.prompt_token_ids is not None
1580
        num_prompt_tokens = len(final_res.prompt_token_ids)
1581
1582
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1583
        num_generated_tokens = sum(
1584
1585
1586
1587
1588
1589
1590
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1591
1592
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1593
1594
                cached_tokens=final_res.num_cached_tokens
            )
1595
1596
1597

        request_metadata.final_usage_info = usage

1598
1599
1600
1601
1602
1603
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1604
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1605
1606
1607
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1608
            kv_transfer_params=final_res.kv_transfer_params,
1609
1610
        )

1611
1612
1613
1614
1615
1616
1617
1618
1619
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1620
1621
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1622
1623
                            tc.function, "arguments"
                        ):
1624
                            tool_call_descriptions.append(
1625
1626
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1627
1628
1629
1630
1631
1632
1633
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1634
                        output_token_ids = final_res.outputs[choice.index].token_ids
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1645
        return response
1646
1647

    def _get_top_logprobs(
1648
1649
        self,
        logprobs: dict[int, Logprob],
1650
        top_logprobs: int | None,
1651
        tokenizer: TokenizerLike,
1652
1653
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1654
        return [
1655
            ChatCompletionLogProb(
1656
1657
1658
1659
1660
1661
1662
1663
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1664
1665
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1666
1667
            )
            for i, p in enumerate(logprobs.items())
1668
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1669
1670
1671
1672
1673
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1674
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1675
        tokenizer: TokenizerLike,
1676
1677
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1678
1679
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1680
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1681

1682
1683
1684
1685
1686
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1687
1688
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1689
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1690
                if should_return_as_token_id:
1691
                    token = f"token_id:{token_id}"
1692
1693
                else:
                    token = tokenizer.decode(token_id)
1694

1695
1696
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1697
                        token=token,
1698
                        bytes=list(token.encode("utf-8", errors="replace")),
1699
1700
                    )
                )
1701
            else:
1702
1703
1704
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1705
1706
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1707
                        token=self._get_decoded_token(
1708
1709
1710
                            step_token,
                            token_id,
                            tokenizer,
1711
                            should_return_as_token_id,
1712
1713
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1714
1715
1716
1717
1718
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1719
                        top_logprobs=self._get_top_logprobs(
1720
1721
1722
1723
1724
1725
1726
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1727
1728

        return ChatCompletionLogProbs(content=logprobs_content)
1729

1730
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1731
1732
1733
1734
1735
1736
1737
1738
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1739
1740
1741
1742
1743
1744
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1745
1746
1747

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1748
        delta_message: DeltaMessage | None,
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1760
            output.finish_reason is not None
1761
1762
1763
1764
1765
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1766
1767
1768
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1769
1770
1771
1772
1773
1774
1775

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
    ):
        messages: list[OpenAIMessage] = []

1776
1777
1778
1779
1780
        # because of issues with pydantic we need to potentially
        # re-serialize the tool_calls field of the request
        # for more info: see comment in `maybe_serialize_tool_calls`
        maybe_serialize_tool_calls(request)

1781
1782
1783
1784
1785
1786
1787
1788
        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1789
            python_description=None,
1790
1791
            with_custom_tools=request.tools is not None,
        )
1792
1793
1794
        messages.append(sys_msg)

        # Add developer message.
1795
        dev_msg = get_developer_message(tools=request.tools)
1796
1797
1798
        messages.append(dev_msg)

        # Add user message.
1799
        messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1800
1801
1802
1803

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1804
1805
1806
1807
1808

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1809
        return messages, [prompt_token_ids], [engine_prompt]