serving_chat.py 77.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
22
23
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
24
from vllm.entrypoints.harmony_utils import (
25
26
27
28
29
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
    parse_chat_output,
30
    parse_input_to_harmony_message,
31
32
    render_for_completion,
)
33
from vllm.entrypoints.logger import RequestLogger
34
from vllm.entrypoints.openai.protocol import (
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
55
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
56
from vllm.entrypoints.openai.tool_parsers import ToolParser
57
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
58
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
59
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
60
from vllm.logger import init_logger
61
from vllm.logprobs import Logprob
62
from vllm.outputs import CompletionOutput, RequestOutput
63
from vllm.sampling_params import BeamSearchParams, SamplingParams
64
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
65
66
67
68
69
from vllm.transformers_utils.tokenizers import (
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
70
from vllm.utils.collection_utils import as_list
71
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
72
73
74
75
76

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
77
78
79
    def __init__(
        self,
        engine_client: EngineClient,
80
        models: OpenAIServingModels,
81
82
        response_role: str,
        *,
83
84
        request_logger: RequestLogger | None,
        chat_template: str | None,
85
        chat_template_content_format: ChatTemplateContentFormatOption,
86
        trust_request_chat_template: bool = False,
87
        return_tokens_as_token_ids: bool = False,
88
        reasoning_parser: str = "",
89
        enable_auto_tools: bool = False,
90
        exclude_tools_when_tool_choice_none: bool = False,
91
        tool_parser: str | None = None,
92
        enable_prompt_tokens_details: bool = False,
93
        enable_force_include_usage: bool = False,
94
        enable_log_outputs: bool = False,
95
        log_error_stack: bool = False,
96
    ) -> None:
97
98
99
100
101
102
103
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
104

105
        self.response_role = response_role
106
107
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
108
        self.trust_request_chat_template = trust_request_chat_template
109
        self.enable_log_outputs = enable_log_outputs
110

111
112
113
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

114
115
116
117
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
118
119
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
120
121
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
122
123
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
124

125
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
126
        self.enable_force_include_usage = enable_force_include_usage
127
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
128
        if self.default_sampling_params:
129
130
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
131
132
133
134
135
136
137
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
138
        else:
139
            self.tool_call_id_type = "random"
140

141
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
142
143
144
145
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
146
147
                get_stop_tokens_for_assistant_actions()
            )
148
149
150
151
152
153
154
155
156
157
158

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

159
    async def create_chat_completion(
160
161
        self,
        request: ChatCompletionRequest,
162
163
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
164
165
        """
        Chat Completion API similar to OpenAI's API.
166

167
168
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
169
        Chat Completion API.
170
171
172
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
173
            logger.error("Error with model %s", error_check_ret)
174
175
            return error_check_ret

176
177
178
179
180
181
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

182
        try:
183
            lora_request = self._maybe_get_adapters(
184
185
                request, supports_default_mm_loras=True
            )
186

187
            model_name = self.models.model_name(lora_request)
188

189
            tokenizer = await self.engine_client.get_tokenizer()
190

191
192
            tool_parser = self.tool_parser

193
            if isinstance(tokenizer, MistralTokenizer):
194
195
196
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
197
                maybe_serialize_tool_calls(request)
198
                truncate_tool_call_ids(request)
199
                validate_request_params(request)
200

201
202
203
204
205
206
            if (
                request.tool_choice == "auto"
                and not (self.enable_auto_tools and tool_parser is not None)
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
            ):
207
208
209
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
210
                    '"auto" tool choice requires '
211
212
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
213

214
215
216
217
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
218
219
220
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
221

222
223
            if not self.use_harmony:
                # Common case.
224
225
226
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
227
                    trust_request_chat_template=self.trust_request_chat_template,
228
229
230
                )
                if error_check_ret is not None:
                    return error_check_ret
231
232
233
234
235
236
237
238
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = await self._preprocess_chat(
                    request,
                    tokenizer,
                    request.messages,
239
                    chat_template=request.chat_template or self.chat_template,
240
                    chat_template_content_format=self.chat_template_content_format,
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = self._make_request_with_harmony(request)
256
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
257
            logger.exception("Error in preprocessing prompt inputs")
258
            return self.create_error_response(f"{e} {e.__cause__}")
259

260
261
262
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
263
264
265
266
267

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

268
269
270
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

271
        # Schedule the request and get the result generator.
272
        generators: list[AsyncGenerator[RequestOutput, None]] = []
273
        try:
274
            for i, engine_prompt in enumerate(engine_prompts):
275
                prompt_text, _, _ = self._get_prompt_components(request_prompts[i])
276
277
278
279
280
281
282
283

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
284
285
                    default_sampling_params=self.default_sampling_params,
                )
286

287
                sampling_params: SamplingParams | BeamSearchParams
288
289
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
290
291
                        max_tokens, self.default_sampling_params
                    )
292
293
                else:
                    sampling_params = request.to_sampling_params(
294
295
296
297
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
298
299
300
301
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
302

303
304
305
306
307
308
                self._log_inputs(
                    request_id,
                    request_prompts[i],
                    params=sampling_params,
                    lora_request=lora_request,
                )
309

310
311
312
313
314
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
315
316

                if isinstance(sampling_params, BeamSearchParams):
317
                    generator = self.beam_search(
318
319
320
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
321
                        lora_request=lora_request,
322
323
                    )
                else:
324
325
326
327
328
329
330
331
                    engine_request, tokenization_kwargs = await self._process_inputs(
                        request_id,
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
                    )
332

333
                    generator = self.engine_client.generate(
334
                        engine_request,
335
336
337
338
339
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
340
341
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
342
                        data_parallel_rank=data_parallel_rank,
343
344
345
                    )

                generators.append(generator)
346
        except ValueError as e:
347
            # TODO: Use a vllm-specific Validation Error
348
349
            return self.create_error_response(str(e))

350
        assert len(generators) == 1
351
        (result_generator,) = generators
352

353
354
355
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
356
357
358
359
360
361
362
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
363
            )
364

365
366
        try:
            return await self.chat_completion_full_generator(
367
368
369
370
371
372
373
374
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
375
376
377
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
378
379
380
381

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
382
        return request.messages[-1]["role"]
383

384
    @staticmethod
385
    def _bracket_level(s: str, opening="{", closing="}") -> int:
386
387
388
389
390
391
392
393
394
395
396
397
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
398
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
399
400
401
402
403
404
405
406
407
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
408
            if c == "{":
409
410
                bracket_level += 1
                passed_zero = bracket_level == 0
411
            elif c == "}":
412
413
414
415
416
417
418
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
419
                if c == ",":
420
421
422
423
424
425
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
426
        current_text: str | None,
427
428
        delta_text: str,
        function_name_returned: bool,
429
430
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
431
432
433
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
434
435
436
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
437
            logger.debug("not enough tokens to parse into JSON yet")
438
439
440
441
442
443
444
445
446
447
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
448
449
                delta_text, previous_text
            )
450
451
452
453
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
454
455
456
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
457
458
459
460
461
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
462
463
464
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
465
466
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
467
468
                        arguments, previous_text
                    )
469
470
471
472

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
473
                    if finishes_previous_tool and "parameters" not in current_tool_call:
474
475
476
                        current_tool_call = obj[-2]

                    function_name_returned = True
477
478
479
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
480
481
482
483
484
485
486
487
488
489
490
491
492
493
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
494
495
496

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
497
498
                        delta_text, previous_text
                    )
499
500

                    if delta_text != "":
501
502
503
504
505
506
507
508
509
510
511
512
513
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
514
515
516
517
518
                    else:
                        delta_message = None

        return delta_message, function_name_returned

519
    async def chat_completion_stream_generator(
520
521
522
523
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
524
        model_name: str,
525
        conversation: list[ConversationMessage],
526
        tokenizer: AnyTokenizer,
527
        request_metadata: RequestResponseMetadata,
528
    ) -> AsyncGenerator[str, None]:
529
        created_time = int(time.time())
530
        chunk_object_type: Final = "chat.completion.chunk"
531
        first_iteration = True
532
533

        # Send response for each token for each request.n (index)
534
535
536
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
537
        num_prompt_tokens = 0
538
        num_cached_tokens = None
539
540
        if self.use_harmony:
            harmony_parsers = [
541
                get_streamable_parser_for_assistant() for _ in range(num_choices)
542
            ]
543
544
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
545
546
547
548
549
550
551
552
553

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
554
555
            and self._should_stream_with_auto_tool_parsing(request)
        )
556

557
        all_previous_token_ids: list[list[int]] | None
558
        function_name_returned = [False] * num_choices
559
        if self.tool_call_id_type == "kimi_k2":
560
561
562
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
563

564
565
566
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

567
568
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
569
        if tool_choice_auto or self.reasoning_parser:
570
571
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
572
573
574
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
575
        else:
576
            all_previous_token_ids = None
577

578
        try:
579
            if self.reasoning_parser:
580
581
582
583
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
                    chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                )
584
585
586
587
588
589
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
590
591
592
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
593
                tool_parsers: list[ToolParser | None] = [
594
595
596
597
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
598
        except Exception as e:
599
            logger.exception("Error in tool parser creation.")
600
601
602
603
604
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

605
        stream_options = request.stream_options
606
607
608
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
609

610
611
        try:
            async for res in result_generator:
612
613
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
614
615
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
616

617
618
619
620
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
621
                    num_cached_tokens = res.num_cached_tokens
622
623
                    # Send first response for each request.n (index) with
                    # the role
624
                    role = self.get_chat_request_role(request)
625
626
627

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
628
                    for i in range(num_choices):
629
630
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
631
632
633
634
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
635
                            logprobs=None,
636
637
                            finish_reason=None,
                        )
638
639

                        # return prompt_token_ids at the first chunk ever
640
641
642
643
644
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
645
                            model=model_name,
646
647
648
649
650
651
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
652

653
654
655
656
657
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
658
659
                                total_tokens=num_prompt_tokens,
                            )
660

661
662
663
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

664
665
                    # Send response to echo the input portion of the
                    # last message
666
                    if request.echo:
667
                        last_msg_content: str | list[dict[str, str]] = ""
668
669
670
671
672
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
673
                            last_msg_content = conversation[-1]["content"] or ""
674
675

                        if last_msg_content:
676
                            for i in range(num_choices):
677
678
679
680
681
682
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
683
684
685
686
687
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
688
689
                                    model=model_name,
                                )
690
691
692
693
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
694
695
                                        total_tokens=num_prompt_tokens,
                                    )
696

697
                                data = chunk.model_dump_json(exclude_unset=True)
698
699
700
701
702
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
703
                    tool_parser = tool_parsers[i]
704
705
706
707

                    if finish_reason_sent[i]:
                        continue

708
                    if request.logprobs and request.top_logprobs is not None:
709
                        assert output.logprobs is not None, "Did not output logprobs"
710
                        logprobs = self._create_chat_logprobs(
711
712
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
713
                            tokenizer=tokenizer,
714
                            num_output_top_logprobs=request.top_logprobs,
715
                            return_as_token_id=request.return_tokens_as_token_ids,
716
717
718
719
                        )
                    else:
                        logprobs = None

720
721
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
722
                        prev_recipient = harmony_parser.current_recipient
723
                        delta_text = ""
724
725
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
726
                            delta_text += harmony_parser.last_content_delta or ""
727
728
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
729
730
                    else:
                        delta_text = output.text
731

732
733
734
735
736
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
737
738
739
                        # Chunked prefill case, don't return empty chunks
                        continue

740
                    delta_message: DeltaMessage | None
741

742
                    # just update previous_texts and previous_token_ids
743
                    if tool_choice_auto or self.reasoning_parser:
744
745
746
747
748
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
749
750
                        # avoid the None + list error.
                        if previous_token_ids:
751
                            current_token_ids = previous_token_ids + as_list(
752
753
                                output.token_ids
                            )
754
                        else:
755
                            current_token_ids = as_list(output.token_ids)
756

757
                    if self.use_harmony:
758
                        if cur_channel == "final":
759
                            delta_message = DeltaMessage(content=delta_text)
760
761
762
                        elif cur_channel == "analysis":
                            if request.include_reasoning:
                                delta_message = DeltaMessage(
763
764
                                    reasoning_content=delta_text
                                )
765
766
                            else:
                                delta_message = None
767
768
769
770
771
                        elif (
                            cur_channel == "commentary"
                            and cur_recipient
                            and cur_recipient.startswith("functions.")
                        ):
772
773
774
                            # Count completed tool calls to determine index
                            base_index = 0
                            for msg in harmony_parser.messages:
775
776
777
778
779
                                if (
                                    msg.channel == "commentary"
                                    and msg.recipient
                                    and msg.recipient.startswith("functions.")
                                ):
780
781
782
                                    base_index += 1

                            if prev_recipient != cur_recipient:
783
784
785
786
787
788
789
790
791
792
793
794
795
796
                                tool_name = cur_recipient.split("functions.", 1)[1]
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            id=make_tool_call_id(),
                                            type="function",
                                            function=DeltaFunctionCall(
                                                name=tool_name,
                                                arguments="",
                                            ),
                                            index=base_index,
                                        )
                                    ]
                                )
797
                            elif delta_text:
798
799
800
801
802
803
804
805
806
807
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            index=base_index,
                                            function=DeltaFunctionCall(
                                                arguments=delta_text
                                            ),
                                        )
                                    ]
                                )
808
809
810
811
812
813
814
                            else:
                                delta_message = None

                            if delta_message is not None:
                                harmony_tools_streamed[i] = True
                        else:
                            delta_message = None
815
                    # handle streaming deltas for tools with named tool_choice
816
                    elif tool_choice_function_name:
817
818
819
820
821
822
823
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
824
825
                            assert reasoning_parser is not None
                            delta_message = (
826
                                reasoning_parser.extract_reasoning_content_streaming(
827
828
829
830
831
832
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
833
834
                                )
                            )
835
836
837
838
839
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Only keep 'content', remove 'reasoning_content'.
840
                            if reasoning_parser.is_reasoning_end(
841
842
843
844
845
846
847
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
848
                                reasoning_end_arr[i] = True
849
850
851
852
853
854
855
856
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
857
                            if self.reasoning_parser:
858
859
860
                                delta_text = previous_text + delta_text
                                current_text = ""

861
862
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
863
864
865
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
866
867
                            else:
                                delta_tool_call = DeltaToolCall(
868
                                    id=make_tool_call_id(),
869
870
871
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
872
873
874
875
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
876
877
                                function_name_returned[i] = True

878
879
880
881
882
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
883
                            tools_streamed[i] = True
884

885
886
887
888
889
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
890
891
892
893
894
895
896
897
898
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
                            and res.prompt_token_ids
                            and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        ):
                            reasoning_end_arr[i] = True
899

900
901
902
903
904
905
906
907
908
909
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
                                reasoning_parser.extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
910
                            )
911
912
913
914
915
916
917
918
919
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

920
                        else:
921
                            # either finished reasoning or no reasoning at all
922
                            content = current_text
923
924
925
926
927
928
929
930
931

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
932
                            )
933
934
935
936
937
938
939
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
940

941
942
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
943
                    elif tool_choice_auto and self.reasoning_parser:
944
945
946
947
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
948
                        output_token_ids = as_list(output.token_ids)
949
950
                        if not reasoning_end_arr[i]:
                            delta_message = (
951
                                reasoning_parser.extract_reasoning_content_streaming(
952
953
954
955
956
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
957
                                    output_token_ids,
958
959
                                )
                            )
960
961
962
963
964
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
965
966
967
968
969
970
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
971
                                reasoning_end_arr[i] = True
972
                                current_token_ids = output_token_ids
973
974
975
976
977
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
978
979
980
981
                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
982
                            if reasoning_parser.is_reasoning_end(output_token_ids):
983
                                reasoning_end_arr[i] = True
984
                                current_token_ids = (
985
                                    reasoning_parser.extract_content_ids(
986
987
988
                                        output_token_ids
                                    )
                                )
989
990
991
992
993
994
995
996
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
997
                            delta_token_ids = output_token_ids
998
999
1000
1001
1002
1003
1004
1005
1006
1007
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1008
                            delta_message = tool_parser.extract_tool_calls_streaming(
1009
1010
                                previous_text=previous_text,
                                current_text=current_text,
1011
                                delta_text=delta_text,
1012
1013
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1031
1032
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1033

1034
                    # when only reasoning
1035
                    elif self.reasoning_parser:
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
                        delta_message = (
                            reasoning_parser.extract_reasoning_content_streaming(
                                previous_text,
                                current_text,
                                delta_text,
                                previous_token_ids,
                                current_token_ids,
                                output.token_ids,
                            )
                        )
1046
                    # handle streaming just a content delta
1047
1048
1049
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1050
                    # update the previous values for the next iteration
1051
1052
1053
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1054
1055
1056
1057
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1058
1059
1060
1061
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1062

1063
                    # set the previous values for the next iteration
1064
                    previous_num_tokens[i] += len(output.token_ids)
1065
1066
1067
1068
1069
1070

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1071
1072
1073
1074
                        if output.finish_reason is None:
                            continue
                        else:
                            delta_message = DeltaMessage()
1075

1076
1077
1078
1079
1080
1081
1082
1083
1084
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1085
1086
                                if tc.function and tc.function.arguments
                            )
1087
1088
1089
1090
1091

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1092
                                output_token_ids=as_list(output.token_ids),
1093
1094
1095
1096
1097
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1098
1099
1100
1101
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1102
                            delta=delta_message,
1103
                            logprobs=logprobs,
1104
                            finish_reason=None,
1105
1106
1107
1108
1109
1110
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1111
1112

                    # if the model is finished generating
1113
                    else:
1114
1115
1116
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1117
                        # only happens if we are NOT using structured outputs
1118
                        auto_tools_called = False
1119
                        if tool_parser:
1120
1121
1122
1123
1124
1125
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1126
1127
1128
                        else:
                            index = 0

1129
1130
1131
1132
1133
1134
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1135
                            latest_delta_len = 0
1136
1137
                            if (
                                isinstance(
1138
                                    delta_message.tool_calls[0].function,
1139
1140
1141
1142
1143
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1144
                                latest_delta_len = len(
1145
1146
                                    delta_message.tool_calls[0].function.arguments
                                )
1147

1148
1149
1150
1151
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1152
1153
1154
1155
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1156

1157
                            # get what we've streamed so far for arguments
1158
                            # for the current tool
1159
1160
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1161
                                actual_call = actual_call[:-latest_delta_len]
1162
1163

                            # check to see if there's anything left to stream
1164
                            remaining_call = expected_call.replace(actual_call, "", 1)
1165
                            # set that as a delta message
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    DeltaToolCall(
                                        index=index,
                                        function=DeltaFunctionCall(
                                            arguments=remaining_call
                                        ).model_dump(exclude_none=True),
                                    )
                                ]
                            )
1176

1177
                        # Send the finish response for each request.n only once
1178
1179
1180
1181
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1182
1183
                        if (
                            auto_tools_called
1184
                            or (tools_streamed[i] and not tool_choice_function_name)
1185
1186
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1187
1188
                            finish_reason_ = "tool_calls"
                        else:
1189
1190
1191
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1192
1193
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1194
                            delta=delta_message,
1195
                            logprobs=logprobs,
1196
                            finish_reason=finish_reason_,
1197
                            stop_reason=output.stop_reason,
1198
1199
1200
1201
1202
1203
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1204

1205
                        finish_reason_sent[i] = True
1206

1207
1208
1209
1210
1211
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1212
1213
                        model=model_name,
                    )
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1224
                    data = chunk.model_dump_json(exclude_unset=True)
1225
1226
                    yield f"data: {data}\n\n"

1227
1228
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1229
1230
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1231
1232
1233
1234
1235
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1236
1237
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1238
1239
                        cached_tokens=num_cached_tokens
                    )
1240
1241
1242
1243
1244
1245
1246

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1247
1248
1249
1250
1251
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1252
                yield f"data: {final_usage_data}\n\n"
1253

1254
1255
1256
1257
1258
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1259
1260
1261
1262
1263
1264
1265
1266
1267
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1268
1269
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1270
1271
1272
1273
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1274
                        output_token_ids=None,  # Consider also logging all token IDs
1275
1276
1277
1278
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1279

1280
        except Exception as e:
1281
            # TODO: Use a vllm-specific Validation Error
1282
            logger.exception("Error in chat completion stream generator.")
1283
1284
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1285
1286
1287
1288
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1289
1290
1291
1292
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1293
        model_name: str,
1294
        conversation: list[ConversationMessage],
1295
        tokenizer: AnyTokenizer,
1296
        request_metadata: RequestResponseMetadata,
1297
    ) -> ErrorResponse | ChatCompletionResponse:
1298
        created_time = int(time.time())
1299
        final_res: RequestOutput | None = None
1300

1301
1302
1303
1304
1305
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1306
1307
1308
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1309

1310
1311
        assert final_res is not None

1312
        choices: list[ChatCompletionResponseChoice] = []
1313
        if self.tool_call_id_type == "kimi_k2":
1314
1315
1316
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1317

1318
1319
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1320
            token_ids = output.token_ids
1321
            out_logprobs = output.logprobs
1322
            tool_call_info = None
1323

1324
1325
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1326
                logprobs = self._create_chat_logprobs(
1327
                    token_ids=token_ids,
1328
                    top_logprobs=out_logprobs,
1329
                    num_output_top_logprobs=request.top_logprobs,
1330
                    tokenizer=tokenizer,
1331
                    return_as_token_id=request.return_tokens_as_token_ids,
1332
1333
1334
                )
            else:
                logprobs = None
1335
1336

            if self.use_harmony:
1337
1338
1339
1340
                reasoning_content, content, _ = parse_chat_output(token_ids)
                if not request.include_reasoning:
                    reasoning_content = None

1341
1342
1343
1344
1345
1346
1347
1348
                if self.tool_parser is not None:
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1349
                    content = tool_call_info.content
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                    )
1362
1363
1364
1365
1366

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1367
1368
1369
1370
1371
1372
1373
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1374
                    stop_reason=output.stop_reason,
1375
1376
1377
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1378
1379
1380
                )
                choices.append(choice_data)
                continue
1381

1382
            if self.reasoning_parser:
1383
                try:
1384
1385
1386
1387
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
                        chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                    )
1388
1389
1390
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1391
1392
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1393
1394
1395
                reasoning_content, content = reasoning_parser.extract_reasoning_content(
                    output.text, request=request
                )
1396
1397
                if not request.include_reasoning:
                    reasoning_content = None
1398
1399
1400
            else:
                reasoning_content = None
                content = output.text
1401

1402
            auto_tools_called = False
1403
1404
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1415
1416
1417
1418
1419
1420
1421
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1422
1423

            # if the request uses tools and specified a tool choice
1424
1425
1426
1427
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1428
                assert tool_calls is not None and len(tool_calls) > 0
1429
1430
                message = ChatMessage(
                    role=role,
1431
                    reasoning_content=reasoning_content,
1432
                    content="",
1433
                    tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1434
                )
1435

1436
            elif request.tool_choice and request.tool_choice == "required":
1437
1438
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1439
                for tool_call in tool_calls:
1440
1441
1442
1443
1444
1445
1446
1447
                    tool_call_class_items.append(
                        tool_call_class(
                            id=make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
                                idx=history_tool_call_cnt,
                            ),
                            function=tool_call,
1448
1449
                        )
                    )
1450
                    history_tool_call_cnt += 1
1451
1452
1453
                message = ChatMessage(
                    role=role,
                    content="",
1454
                    tool_calls=tool_call_class_items,
1455
1456
                    reasoning_content=reasoning_content,
                )
1457

1458
1459
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1460
            elif not request.tool_choice or request.tool_choice == "none":
1461
1462
1463
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1464
1465

            # handle when there are tools and tool choice is auto
1466
1467
1468
1469
1470
1471
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1472
1473
1474
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1475
1476
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1477
1478
1479
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
1480
1481
1482
1483
1484
1485
1486
1487
                        content=content,
                        tool_calls=[
                            ToolCall(
                                function=tc,
                                type="function",
                            )
                            for tc in tool_calls
                        ],
1488
                    )
1489
1490
1491
1492

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1493
1494
1495
1496
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1497
1498
                    if content and len(content) > 0:
                        ret_content = content
1499
1500
1501
1502
1503
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=ret_content,
                    )
1504
1505
1506
1507
1508
1509

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1510
1511
1512
1513
1514
                    "completion."
                )
                message = ChatMessage(
                    role=role, reasoning_content=reasoning_content, content=content
                )
1515
1516
1517
1518
1519
1520
1521
1522
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1523

1524
1525
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1526
                message=message,
1527
                logprobs=logprobs,
1528
1529
1530
1531
1532
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1533
                stop_reason=output.stop_reason,
1534
1535
1536
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1537
            )
1538

1539
1540
            choices.append(choice_data)

1541
        if request.echo:
1542
            last_msg_content: str | list[dict[str, str]] = ""
1543
1544
1545
1546
1547
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1548
                last_msg_content = conversation[-1]["content"] or ""
1549
            if isinstance(last_msg_content, list):
1550
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1551
1552

            for choice in choices:
1553
                full_message = last_msg_content + (choice.message.content or "")
1554
1555
                choice.message.content = full_message

1556
        assert final_res.prompt_token_ids is not None
1557
        num_prompt_tokens = len(final_res.prompt_token_ids)
1558
1559
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1560
        num_generated_tokens = sum(
1561
1562
1563
1564
1565
1566
1567
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1568
1569
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1570
1571
                cached_tokens=final_res.num_cached_tokens
            )
1572
1573
1574

        request_metadata.final_usage_info = usage

1575
1576
1577
1578
1579
1580
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1581
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1582
1583
1584
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1585
            kv_transfer_params=final_res.kv_transfer_params,
1586
1587
        )

1588
1589
1590
1591
1592
1593
1594
1595
1596
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1597
1598
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1599
1600
                            tc.function, "arguments"
                        ):
1601
                            tool_call_descriptions.append(
1602
1603
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1604
1605
1606
1607
1608
1609
1610
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1611
                        output_token_ids = final_res.outputs[choice.index].token_ids
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1622
        return response
1623
1624

    def _get_top_logprobs(
1625
1626
        self,
        logprobs: dict[int, Logprob],
1627
        top_logprobs: int | None,
1628
1629
1630
        tokenizer: AnyTokenizer,
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1631
        return [
1632
            ChatCompletionLogProb(
1633
1634
1635
1636
1637
1638
1639
1640
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1641
1642
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1643
1644
            )
            for i, p in enumerate(logprobs.items())
1645
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1646
1647
1648
1649
1650
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1651
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1652
        tokenizer: AnyTokenizer,
1653
1654
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1655
1656
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1657
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1658

1659
1660
1661
1662
1663
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1664
1665
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1666
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1667
                if should_return_as_token_id:
1668
                    token = f"token_id:{token_id}"
1669
1670
                else:
                    token = tokenizer.decode(token_id)
1671

1672
1673
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1674
                        token=token,
1675
                        bytes=list(token.encode("utf-8", errors="replace")),
1676
1677
                    )
                )
1678
            else:
1679
1680
1681
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1682
1683
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1684
                        token=self._get_decoded_token(
1685
1686
1687
                            step_token,
                            token_id,
                            tokenizer,
1688
                            should_return_as_token_id,
1689
1690
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1691
1692
1693
1694
1695
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1696
                        top_logprobs=self._get_top_logprobs(
1697
1698
1699
1700
1701
1702
1703
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1704
1705

        return ChatCompletionLogProbs(content=logprobs_content)
1706

1707
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1708
1709
1710
1711
1712
1713
1714
1715
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1716
1717
1718
1719
1720
1721
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1722
1723
1724

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1725
        delta_message: DeltaMessage | None,
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1737
            output.finish_reason is not None
1738
1739
1740
1741
1742
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1743
1744
1745
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
    ):
        messages: list[OpenAIMessage] = []

        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1761
            python_description=None,
1762
1763
            with_custom_tools=request.tools is not None,
        )
1764
1765
1766
        messages.append(sys_msg)

        # Add developer message.
1767
        dev_msg = get_developer_message(tools=request.tools)
1768
1769
1770
1771
        messages.append(dev_msg)

        # Add user message.
        for chat_msg in request.messages:
1772
            messages.extend(parse_input_to_harmony_message(chat_msg))
1773
1774
1775
1776

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1777
1778
1779
1780
1781

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1782
        return messages, [prompt_token_ids], [engine_prompt]