serving_chat.py 70.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
9
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Callable, Final, Optional, Union
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
from pydantic import TypeAdapter
17

18
from vllm.config import ModelConfig
19
from vllm.engine.protocol import EngineClient
20
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
21
                                         ConversationMessage,
22
23
                                         get_history_tool_calls_cnt,
                                         make_tool_call_id)
24
25
26
27
from vllm.entrypoints.harmony_utils import (
    get_developer_message, get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant, get_system_message, parse_chat_input,
    parse_chat_output, render_for_completion)
28
from vllm.entrypoints.logger import RequestLogger
29
from vllm.entrypoints.openai.protocol import (
30
31
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
32
    ChatCompletionRequest, ChatCompletionResponse,
33
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
34
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
35
36
    DeltaToolCall, ErrorResponse, FunctionCall, FunctionDefinition,
    PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo)
37
38
from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
                                                    clamp_prompt_logprobs)
39
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
40
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
41
42
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
    MistralToolCall)
43
from vllm.entrypoints.utils import get_max_tokens
44
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
45
from vllm.logger import init_logger
46
from vllm.outputs import CompletionOutput, RequestOutput
47
from vllm.reasoning import ReasoningParser, ReasoningParserManager
48
from vllm.sampling_params import BeamSearchParams, SamplingParams
49
from vllm.sequence import Logprob
50
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
51
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
52
53
                                                truncate_tool_call_ids,
                                                validate_request_params)
54
from vllm.utils import as_list
55
56
57
58
59
60

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

61
62
63
64
    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
65
        models: OpenAIServingModels,
66
67
68
69
70
71
        response_role: str,
        *,
        request_logger: Optional[RequestLogger],
        chat_template: Optional[str],
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
72
        reasoning_parser: str = "",
73
        enable_auto_tools: bool = False,
74
        exclude_tools_when_tool_choice_none: bool = False,
75
76
        tool_parser: Optional[str] = None,
        enable_prompt_tokens_details: bool = False,
77
        enable_force_include_usage: bool = False,
78
        enable_log_outputs: bool = False,
79
    ) -> None:
80
        super().__init__(engine_client=engine_client,
81
                         model_config=model_config,
82
                         models=models,
83
                         request_logger=request_logger,
84
85
                         return_tokens_as_token_ids=return_tokens_as_token_ids,
                         enable_force_include_usage=enable_force_include_usage)
86

87
        self.response_role = response_role
88
89
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
90
        self.enable_log_outputs = enable_log_outputs
91

92
93
94
95
96
97
98
99
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

100
101
        self.reasoning_parser: Optional[Callable[[AnyTokenizer],
                                                 ReasoningParser]] = None
102
        if reasoning_parser:
103
104
105
106
            try:
                self.reasoning_parser = (
                    ReasoningParserManager.get_reasoning_parser(
                        reasoning_parser))
107
                assert self.reasoning_parser is not None
108
            except Exception as e:
109
110
                raise TypeError(
                    f"{reasoning_parser=} has not been registered") from e
111
112
        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
113
            try:
114
115
116
117
118
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
119
120
121
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
122
                raise TypeError("Error: --enable-auto-tool-choice requires "
123
124
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
125
126
        self.exclude_tools_when_tool_choice_none = (
            exclude_tools_when_tool_choice_none)
127

128
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
129
        self.enable_force_include_usage = enable_force_include_usage
130
131
132
        self.default_sampling_params = (
            self.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
133
134
135
136
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
            logger.info("Using default chat sampling params from %s: %s",
                        source, self.default_sampling_params)
137
138
139
140
        if self.model_config.hf_config.model_type == 'kimi_k2':
            self.tool_call_id_type = 'kimi_k2'
        else:
            self.tool_call_id_type = 'random'
141

142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
        self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
                get_stop_tokens_for_assistant_actions())

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

159
    async def create_chat_completion(
160
161
        self,
        request: ChatCompletionRequest,
162
163
164
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
165
166
        """
        Chat Completion API similar to OpenAI's API.
167

168
169
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
170
        Chat Completion API.
171
172
173
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
174
            logger.error("Error with model %s", error_check_ret)
175
176
            return error_check_ret

177
178
179
180
181
182
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

183
        try:
184
185
            lora_request = self._maybe_get_adapters(
                request, supports_default_mm_loras=True)
186

187
            model_name = self._get_model_name(request.model, lora_request)
188

189
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
190

191
192
            tool_parser = self.tool_parser

193
            if isinstance(tokenizer, MistralTokenizer):
194
195
196
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
197
                maybe_serialize_tool_calls(request)
198
                truncate_tool_call_ids(request)
199
                validate_request_params(request)
200

201
202
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
203
204
                    and not isinstance(tokenizer, MistralTokenizer)
                    and not self.use_harmony):
205
206
207
208
209
210
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
211

212
213
214
            if (request.tools is None
                    or (request.tool_choice == "none"
                        and self.exclude_tools_when_tool_choice_none)):
215
216
217
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
218

219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
            if not self.use_harmony:
                # Common case.
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = await self._preprocess_chat(
                    request,
                    tokenizer,
                    request.messages,
                    chat_template=request.chat_template or self.chat_template,
                    chat_template_content_format=self.
                    chat_template_content_format,
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    truncate_prompt_tokens=request.truncate_prompt_tokens,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = self._make_request_with_harmony(request)
248
249
        except (ValueError, TypeError, RuntimeError,
                jinja2.TemplateError) as e:
250
            logger.exception("Error in preprocessing prompt inputs")
251
            return self.create_error_response(f"{e} {e.__cause__}")
252

253
254
        request_id = "chatcmpl-" \
                     f"{self._base_request_id(raw_request, request.request_id)}"
255
256
257
258
259

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

260
        # Schedule the request and get the result generator.
261
        generators: list[AsyncGenerator[RequestOutput, None]] = []
262
        try:
263
264
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
265
266
267
268
269
270
271
272
273
274

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
                    default_sampling_params=self.default_sampling_params)

275
276
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
277
                        max_tokens, self.default_sampling_params)
278
279
                else:
                    sampling_params = request.to_sampling_params(
280
                        max_tokens, self.model_config.logits_processor_pattern,
281
                        self.default_sampling_params)
282
283
284
285

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
286
                                 lora_request=lora_request)
287
288
289
290
291
292
293
294
295

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
296
                        lora_request=lora_request,
297
298
299
300
301
302
303
304
305
306
307
308
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
                    )

                generators.append(generator)
309
        except ValueError as e:
310
            # TODO: Use a vllm-specific Validation Error
311
312
            return self.create_error_response(str(e))

313
314
315
        assert len(generators) == 1
        result_generator, = generators

316
317
318
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
319
320
321
322
323
324
325
326
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
                enable_force_include_usage=self.enable_force_include_usage)
327

328
329
        try:
            return await self.chat_completion_full_generator(
330
331
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
332
333
334
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
335
336
337
338

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
339
        return request.messages[-1]["role"]
340

341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
    @staticmethod
    def _bracket_level(s: str, opening='{', closing='}') -> int:
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
    def _filter_delta_text(delta_text: str,
                           previous_text: str) -> tuple[str, bool]:
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
            if c == '{':
                bracket_level += 1
                passed_zero = bracket_level == 0
            elif c == '}':
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
                if c == ',':
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
384
        current_text: Optional[str],
385
386
        delta_text: str,
        function_name_returned: bool,
387
        tool_call_idx: Optional[int] = None
388
    ) -> tuple[Optional[DeltaMessage], bool]:
389
390
391
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
            logger.debug('not enough tokens to parse into JSON yet')
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
                delta_text, previous_text)
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
            if not finishes_previous_tool and ("name" not in current_tool_call
                                               or "parameters"
                                               not in current_tool_call):
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
                    param_match = re.search(r'.*"parameters":\s*(.*)',
                                            current_text)
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
                        arguments, previous_text)

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
                    if (finishes_previous_tool
                            and "parameters" not in current_tool_call):
                        current_tool_call = obj[-2]

                    function_name_returned = True
433
434
435
436
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
                        idx=tool_call_idx)
437
                    delta_message = DeltaMessage(tool_calls=[
438
                        DeltaToolCall(id=tool_call_id,
439
440
441
                                      function=DeltaFunctionCall(
                                          name=current_tool_call["name"],
                                          arguments=arguments),
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
                                      index=len(obj) - 1,
                                      type="function")
                    ])

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
                        delta_text, previous_text)

                    if delta_text != "":
                        delta_message = DeltaMessage(tool_calls=[
                            DeltaToolCall(
                                function=DeltaFunctionCall(
                                    # OpenAI API returns None
                                    # instead of name every time
                                    name=None,
                                    arguments=delta_text),
458
                                index=len(obj) - 1)
459
460
461
462
463
464
                        ])
                    else:
                        delta_message = None

        return delta_message, function_name_returned

465
    async def chat_completion_stream_generator(
466
467
468
469
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
470
        model_name: str,
471
        conversation: list[ConversationMessage],
472
        tokenizer: AnyTokenizer,
473
        request_metadata: RequestResponseMetadata,
474
        enable_force_include_usage: bool,
475
    ) -> AsyncGenerator[str, None]:
476
        created_time = int(time.time())
477
        chunk_object_type: Final = "chat.completion.chunk"
478
        first_iteration = True
479
480

        # Send response for each token for each request.n (index)
481
482
483
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
484
        num_prompt_tokens = 0
485
        num_cached_tokens = None
486
487
488
489
490
        if self.use_harmony:
            harmony_parsers = [
                get_streamable_parser_for_assistant()
                for _ in range(num_choices)
            ]
491
492
493
494
495
496
497
498
499
500
501

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

502
        all_previous_token_ids: Optional[list[list[int]]]
503
        function_name_returned = [False] * num_choices
504
505
506
507
        if self.tool_call_id_type == 'kimi_k2':
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
508

509
510
511
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

512
513
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
514
        if tool_choice_auto or self.reasoning_parser:
515
516
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
517
518
519
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
520
521
        elif request.tool_choice == "required":
            all_previous_token_ids = None
522
        else:
523
            all_previous_token_ids = None
524

525
        try:
526
            if self.reasoning_parser:
527
528
529
530
531
532
533
                reasoning_parser = self.reasoning_parser(tokenizer)
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
534
535
536
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
537
                tool_parsers: list[Optional[ToolParser]] = [
538
539
540
541
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
542
        except Exception as e:
543
            logger.exception("Error in tool parser creation.")
544
545
546
547
548
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

549
550
        stream_options = request.stream_options
        if stream_options:
551
552
            include_usage = stream_options.include_usage \
                            or enable_force_include_usage
553
554
555
556
557
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

558
559
        try:
            async for res in result_generator:
560
561
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
562
563
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
564

565
566
567
568
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
569
                    num_cached_tokens = res.num_cached_tokens
570
571
                    # Send first response for each request.n (index) with
                    # the role
572
                    role = self.get_chat_request_role(request)
573
574
575

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
576
                    for i in range(num_choices):
577
578
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
579
580
581
582
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
583
584
                            logprobs=None,
                            finish_reason=None)
585
586

                        # return prompt_token_ids at the first chunk ever
587
588
589
590
591
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
592
593
594
595
                            model=model_name,
                            prompt_token_ids=(res.prompt_token_ids
                                              if request.return_token_ids else
                                              None))
596

597
598
599
600
601
602
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
603

604
605
606
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

607
608
                    # Send response to echo the input portion of the
                    # last message
609
                    if request.echo:
610
                        last_msg_content: Union[str, list[dict[str, str]]] = ""
611
612
613
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
614
615

                        if last_msg_content:
616
                            for i in range(num_choices):
617
618
619
620
621
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
622
                                        logprobs=None,
623
                                        finish_reason=None))
624
625
626
627
628
629
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
630
631
632
633
634
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
635

636
637
638
639
640
641
642
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
643
                    tool_parser = tool_parsers[i]
644
645
646
647

                    if finish_reason_sent[i]:
                        continue

648
                    if request.logprobs and request.top_logprobs is not None:
649
                        assert output.logprobs is not None, (
650
                            "Did not output logprobs")
651
                        logprobs = self._create_chat_logprobs(
652
653
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
654
                            tokenizer=tokenizer,
655
                            num_output_top_logprobs=request.top_logprobs,
656
657
                            return_as_token_id=request.
                            return_tokens_as_token_ids,
658
659
660
661
                        )
                    else:
                        logprobs = None

662
663
664
665
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
666
667
668
                        is_reasoning = \
                            harmony_parser.current_channel == "analysis"
                        if not request.include_reasoning and is_reasoning:
669
670
671
672
673
                            # Skip the reasoning content.
                            continue
                        delta_text = harmony_parser.last_content_delta or ""
                    else:
                        delta_text = output.text
674
675
676
677
678
679

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

680
                    delta_message: Optional[DeltaMessage]
681

682
                    # just update previous_texts and previous_token_ids
683
684
                    if ((tool_choice_auto or self.reasoning_parser)
                            and not self.use_harmony):
685
686
687
688
689
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
690
691
                        # avoid the None + list error.
                        if previous_token_ids:
692
                            current_token_ids = previous_token_ids + as_list(
693
694
                                output.token_ids)
                        else:
695
                            current_token_ids = as_list(output.token_ids)
696

697
                    if self.use_harmony:
698
                        if is_reasoning:
699
700
                            delta_message = DeltaMessage(
                                reasoning_content=delta_text)
701
702
                        else:
                            delta_message = DeltaMessage(content=delta_text)
703
                    # handle streaming deltas for tools with named tool_choice
704
                    elif tool_choice_function_name:
705
                        if (self.reasoning_parser and not reasoning_end_arr[i]
706
707
708
709
710
711
712
713
714
715
716
717
718
                                and not reasoning_parser.is_reasoning_end(
                                    previous_token_ids)):
                            assert reasoning_parser is not None
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))
719
720
721
722
723
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Only keep 'content', remove 'reasoning_content'.
724
                            if reasoning_parser.is_reasoning_end(
725
726
727
728
                                    as_list(output.token_ids)) or (
                                        res.prompt_token_ids
                                        and reasoning_parser.is_reasoning_end(
                                            res.prompt_token_ids)):
729
                                reasoning_end_arr[i] = True
730
731
732
733
734
735
736
737
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
738
                            if self.reasoning_parser:
739
740
741
                                delta_text = previous_text + delta_text
                                current_text = ""

742
743
744
745
746
747
748
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        arguments=delta_text),
                                    index=i)
                            else:
                                delta_tool_call = DeltaToolCall(
749
                                    id=make_tool_call_id(),
750
751
752
753
754
755
756
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
                                        arguments=delta_text),
                                    index=i)
                                function_name_returned[i] = True

757
                            delta_message = DeltaMessage(tool_calls=[
758
                                delta_tool_call,
759
760
                            ])

761
762
763
764
765
766
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]

767
768
769
770
771
772
773
774
                        if self.reasoning_parser:
                            _, content = \
                                reasoning_parser.extract_reasoning_content(
                                    current_text,
                                    request
                                )
                        else:
                            content = current_text
775
776
777
                        delta_message, function_name_returned[i] = (
                            self.extract_tool_call_required_streaming(
                                previous_text=previous_text,
778
                                current_text=content,
779
                                delta_text=delta_text,
780
781
782
783
784
                                function_name_returned=fn_name_returned,
                                tool_call_idx=history_tool_call_cnt))
                        if (delta_message and delta_message.tool_calls and
                                delta_message.tool_calls[0].id is not None):
                            history_tool_call_cnt += 1
785
786
787
788

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text

789
790
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
791
                    elif tool_choice_auto and self.reasoning_parser:
792
793
794
795
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
796
                        output_token_ids = as_list(output.token_ids)
797
798
799
800
801
802
803
804
805
                        if not reasoning_end_arr[i]:
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
806
                                    output_token_ids,
807
                                ))
808
809
810
811
812
813
814
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if res.prompt_token_ids and \
                                reasoning_parser.is_reasoning_end(
815
                                    res.prompt_token_ids):
816
                                reasoning_end_arr[i] = True
817
                                current_token_ids = output_token_ids
818
819
820
821
822
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
823
824
825
826
827
                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if reasoning_parser.is_reasoning_end(
828
                                    output_token_ids):
829
830
831
                                reasoning_end_arr[i] = True
                                current_token_ids =  \
                                    reasoning_parser.extract_content_ids(
832
                                        output_token_ids)
833
834
835
836
837
838
839
840
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
841
                            delta_token_ids = output_token_ids
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

                            delta_message = (
                                tool_parser.extract_tool_calls_streaming(
                                    previous_text=previous_text,
                                    current_text=current_text,
                                    delta_text=delta_text,
                                    previous_token_ids=previous_token_ids,
                                    current_token_ids=current_token_ids,
                                    delta_token_ids=delta_token_ids,
                                    request=request))
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
864
865
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
866
867
                                previous_text=previous_text,
                                current_text=current_text,
868
                                delta_text=delta_text,
869
870
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
871
872
                                delta_token_ids=output.token_ids,
                                request=request))
873

874
                    # when only reasoning
875
                    elif self.reasoning_parser:
876
877
878
879
880
881
882
883
884
                        delta_message = (reasoning_parser.
                                         extract_reasoning_content_streaming(
                                             previous_text,
                                             current_text,
                                             delta_text,
                                             previous_token_ids,
                                             current_token_ids,
                                             output.token_ids,
                                         ))
885
                    # handle streaming just a content delta
886
887
888
                    else:
                        delta_message = DeltaMessage(content=delta_text)

889
                    # update the previous values for the next iteration
890
891
                    if ((tool_choice_auto or self.reasoning_parser)
                            and not self.use_harmony):
892
893
894
895
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
896
897
898
899
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
900

901
                    # set the previous values for the next iteration
902
                    previous_num_tokens[i] += len(output.token_ids)
903
904
905
906
907
908
909
910

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
                                if tc.function and tc.function.arguments)

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
926
                                output_token_ids=as_list(output.token_ids),
927
928
929
930
931
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

932
933
934
935
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
936
                            delta=delta_message,
937
                            logprobs=logprobs,
938
939
940
                            finish_reason=None,
                            token_ids=(as_list(output.token_ids)
                                       if request.return_token_ids else None))
941
942

                    # if the model is finished generating
943
                    else:
944
945
946
947
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
948
                        auto_tools_called = False
949
                        if tool_parser:
950
951
952
953
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
954
955
956
957
958
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
959
960
961
962
963
964
965
966
967
968
                            latest_delta_len = 0
                            if ((isinstance(
                                    delta_message.tool_calls[0].function,
                                    DeltaFunctionCall)) and isinstance(
                                        delta_message.tool_calls[0].function.
                                        arguments, str)):
                                latest_delta_len = len(
                                    delta_message.tool_calls[0].function.
                                    arguments)

969
970
971
972
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
973
974
                                    "arguments", {}),
                                ensure_ascii=False)
975

976
                            # get what we've streamed so far for arguments
977
978
979
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]
980
981
                            if (latest_delta_len > 0):
                                actual_call = actual_call[:-latest_delta_len]
982
983
984
985
986
987
988
989
990
991
992
993

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)
                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

994
995
996
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
997
                            delta=delta_message,
998
                            logprobs=logprobs,
999
                            finish_reason=output.finish_reason
1000
                            if not auto_tools_called else "tool_calls",
1001
1002
1003
                            stop_reason=output.stop_reason,
                            token_ids=(as_list(output.token_ids)
                                       if request.return_token_ids else None))
1004

1005
                        finish_reason_sent[i] = True
1006

1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1023
                    data = chunk.model_dump_json(exclude_unset=True)
1024
1025
                    yield f"data: {data}\n\n"

1026
1027
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1028
1029
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1030
1031
1032
1033
1034
1035
1036
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
1048

1049
1050
1051
1052
1053
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
                        if previous_texts and i < len(previous_texts) else
                        f"<streaming_complete: {previous_num_tokens[i]} tokens>"
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
                        output_token_ids=
                        None,  # Consider also logging all token IDs
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1075

1076
        except Exception as e:
1077
            # TODO: Use a vllm-specific Validation Error
1078
            logger.exception("Error in chat completion stream generator.")
1079
1080
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1081
1082
1083
1084
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1085
1086
1087
1088
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1089
        model_name: str,
1090
        conversation: list[ConversationMessage],
1091
        tokenizer: AnyTokenizer,
1092
        request_metadata: RequestResponseMetadata,
1093
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
1094

1095
        created_time = int(time.time())
1096
        final_res: Optional[RequestOutput] = None
1097

1098
1099
1100
1101
1102
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1103
1104
1105
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1106

1107
1108
        assert final_res is not None

1109
        choices: list[ChatCompletionResponseChoice] = []
1110
1111
1112
1113
        if self.tool_call_id_type == 'kimi_k2':
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1114

1115
1116
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1117
            token_ids = output.token_ids
1118
            out_logprobs = output.logprobs
1119

1120
1121
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1122
                logprobs = self._create_chat_logprobs(
1123
                    token_ids=token_ids,
1124
                    top_logprobs=out_logprobs,
1125
                    num_output_top_logprobs=request.top_logprobs,
1126
                    tokenizer=tokenizer,
1127
                    return_as_token_id=request.return_tokens_as_token_ids,
1128
1129
1130
                )
            else:
                logprobs = None
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162

            if self.use_harmony:
                reasoning_content, final_content, is_tool_call = (
                    parse_chat_output(token_ids))
                if not request.include_reasoning:
                    reasoning_content = None

                if is_tool_call:
                    # TODO(woosuk): Implement tool call for gpt-oss.
                    # For now, only Responses API supports tool call for
                    # gpt-oss.
                    raise NotImplementedError(
                        "Tool call in Chat Completion API is not supported "
                        "for gpt-oss yet. Please use Responses API instead.")
                else:
                    # Normal message
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=final_content,
                    )

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
                    finish_reason="tool_calls" if is_tool_call else
                    output.finish_reason if output.finish_reason else "stop",
                    stop_reason=output.stop_reason,
                )
                choices.append(choice_data)
                continue
1163

1164
            if self.reasoning_parser:
1165
1166
1167
1168
1169
                try:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1170
1171
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1172
1173
1174
                reasoning_content, content = (
                    reasoning_parser.extract_reasoning_content(
                        output.text, request=request))
1175
1176
                if not request.include_reasoning:
                    reasoning_content = None
1177
1178
1179
            else:
                reasoning_content = None
                content = output.text
1180

1181
            auto_tools_called = False
1182
1183
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1184
1185
1186
1187
            if (not self.enable_auto_tools or not self.tool_parser) and \
                (not isinstance(request.tool_choice,
                                ChatCompletionNamedToolChoiceParam
                                ) and request.tool_choice != "required"):
1188
1189
1190
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1191
1192
1193

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
1194
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
1195

1196
1197
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall
1198
1199
                message = ChatMessage(
                    role=role,
1200
                    reasoning_content=reasoning_content,
1201
1202
                    content="",
                    tool_calls=[
1203
                        tool_call_class(function=FunctionCall(
1204
                            name=request.tool_choice.function.name,
1205
1206
1207
1208
                            arguments=content,
                        ))
                    ],
                )
1209

1210
1211
1212
1213
1214
1215
            elif request.tool_choice and request.tool_choice == "required":
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall

                # the fields of FunctionDefinition are a superset of the
                # tool call outputs and can be used for parsing
1216
                assert content is not None
1217
                tool_calls = TypeAdapter(
1218
                    list[FunctionDefinition]).validate_json(content)
1219
1220
1221
1222
1223
1224
1225
                tool_call_ids = []
                for tool_call in tool_calls:
                    tool_call_ids.append(
                        make_tool_call_id(id_type=self.tool_call_id_type,
                                          func_name=tool_call.name,
                                          idx=history_tool_call_cnt))
                    history_tool_call_cnt += 1
1226
1227
1228
1229
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
1230
1231
1232
1233
1234
1235
1236
1237
1238
                        tool_call_class(id=tool_call_ids[i],
                                        function=FunctionCall(
                                            name=tool_call.name,
                                            arguments=json.dumps(
                                                tool_call.parameters,
                                                ensure_ascii=False)))
                        for i, tool_call in enumerate(tool_calls)
                    ],
                    reasoning_content=reasoning_content)
1239

1240
1241
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1242
            elif not request.tool_choice or request.tool_choice == "none":
1243

1244
1245
1246
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1247
1248
1249
1250
1251
1252
1253

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

1254
1255
1256
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
1257
                    logger.exception("Error in tool parser creation.")
1258
1259
                    return self.create_error_response(str(e))

1260
                tool_call_info = tool_parser.extract_tool_calls(
1261
                    content if content is not None else "", request=request)
1262
1263
1264
1265
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
1266
1267
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
1268
                                          reasoning_content=reasoning_content,
1269
1270
1271
1272
1273
1274
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1275
1276
1277
1278
1279
1280
1281
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
                    if (tool_call_info.content
                            and len(tool_call_info.content) > 0):
                        ret_content = tool_call_info.content
1282
1283
                    message = ChatMessage(role=role,
                                          reasoning_content=reasoning_content,
1284
                                          content=ret_content)
1285
1286
1287
1288
1289
1290
1291

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
1292
1293
1294
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1295

1296
1297
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1298
                message=message,
1299
                logprobs=logprobs,
1300
                finish_reason="tool_calls" if auto_tools_called else
1301
                output.finish_reason if output.finish_reason else "stop",
1302
1303
1304
1305
                stop_reason=output.stop_reason,
                token_ids=(as_list(output.token_ids)
                           if request.return_token_ids else None),
            )
1306

1307
1308
            choices.append(choice_data)

1309
        if request.echo:
1310
            last_msg_content: Union[str, list[dict[str, str]]] = ""
1311
1312
            if (conversation and "content" in conversation[-1]
                    and conversation[-1].get("role") == role):
1313
                last_msg_content = conversation[-1]["content"] or ""
1314
1315
1316
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
1317
1318

            for choice in choices:
1319
1320
                full_message = last_msg_content + (choice.message.content
                                                   or "")
1321
1322
                choice.message.content = full_message

1323
        assert final_res.prompt_token_ids is not None
1324
        num_prompt_tokens = len(final_res.prompt_token_ids)
1325
1326
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1327
1328
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
1329
1330
1331
1332
1333
1334
1335
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
1336
1337
1338

        request_metadata.final_usage_info = usage

1339
1340
1341
1342
1343
1344
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1345
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1346
1347
            prompt_token_ids=(final_res.prompt_token_ids
                              if request.return_token_ids else None),
Robert Shaw's avatar
Robert Shaw committed
1348
            kv_transfer_params=final_res.kv_transfer_params,
1349
1350
        )

1351
1352
1353
1354
1355
1356
1357
1358
1359
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1360
1361
1362
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
                                tc.function, "arguments"):
1363
                            tool_call_descriptions.append(
1364
                                f"{tc.function.name}({tc.function.arguments})")
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
                        output_token_ids = final_res.outputs[
                            choice.index].token_ids

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1384
        return response
1385
1386

    def _get_top_logprobs(
1387
            self, logprobs: dict[int, Logprob], top_logprobs: Optional[int],
1388
1389
            tokenizer: AnyTokenizer,
            should_return_as_token_id: bool) -> list[ChatCompletionLogProb]:
1390
        return [
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
            ChatCompletionLogProb(
                token=(token := self._get_decoded_token(
                    p[1],
                    p[0],
                    tokenizer,
                    return_as_token_id=should_return_as_token_id,
                )),
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
            ) for i, p in enumerate(logprobs.items())
1401
1402
1403
1404
1405
1406
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1407
        top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
1408
        tokenizer: AnyTokenizer,
1409
        num_output_top_logprobs: Optional[int] = None,
1410
        return_as_token_id: Optional[bool] = None,
1411
1412
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1413
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1414

1415
1416
        should_return_as_token_id = return_as_token_id if \
            return_as_token_id is not None else self.return_tokens_as_token_ids
1417
1418
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1419
1420
            if step_top_logprobs is None or step_top_logprobs.get(
                    token_id) is None:
1421
                token = tokenizer.decode(token_id)
1422
                if should_return_as_token_id:
1423
                    token = f"token_id:{token_id}"
1424

1425
1426
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1427
                        token=token,
1428
1429
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
1430
            else:
1431
1432
1433
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1434
1435
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1436
                        token=self._get_decoded_token(
1437
1438
1439
                            step_token,
                            token_id,
                            tokenizer,
1440
                            should_return_as_token_id,
1441
1442
1443
1444
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
1445
                        top_logprobs=self._get_top_logprobs(
1446
1447
                            step_top_logprobs, num_output_top_logprobs,
                            tokenizer, should_return_as_token_id),
1448
                    ))
1449
1450

        return ChatCompletionLogProbs(content=logprobs_content)
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1480
1481
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
1482
1483
1484
1485
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
    ):
        messages: list[OpenAIMessage] = []

        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
            python_description=None)
        messages.append(sys_msg)

        # Add developer message.
        dev_msg = get_developer_message()
        messages.append(dev_msg)

        # Add user message.
        for chat_msg in request.messages:
            messages.append(parse_chat_input(chat_msg))

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1515
1516
1517
1518
1519

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1520
        return messages, [prompt_token_ids], [engine_prompt]