"vllm/entrypoints/openai/completion/protocol.py" did not exist on "ae1eba6a9a2342c9660731e8a5674447ad35f757"
serving_chat.py 74.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Callable, Final, Optional, Union
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
from pydantic import TypeAdapter
17

18
from vllm.config import ModelConfig
19
from vllm.engine.protocol import EngineClient
20
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
21
                                         ConversationMessage,
22
23
                                         get_history_tool_calls_cnt,
                                         make_tool_call_id)
24
25
26
27
from vllm.entrypoints.harmony_utils import (
    get_developer_message, get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant, get_system_message, parse_chat_input,
    parse_chat_output, render_for_completion)
28
from vllm.entrypoints.logger import RequestLogger
29
from vllm.entrypoints.openai.protocol import (
30
31
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
32
    ChatCompletionRequest, ChatCompletionResponse,
33
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
34
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
35
36
    DeltaToolCall, ErrorResponse, FunctionCall, FunctionDefinition,
    PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo)
37
38
from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
                                                    clamp_prompt_logprobs)
39
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
40
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
41
42
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
    MistralToolCall)
43
from vllm.entrypoints.utils import get_max_tokens
44
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
45
from vllm.logger import init_logger
46
from vllm.logprobs import Logprob
47
from vllm.outputs import CompletionOutput, RequestOutput
48
from vllm.reasoning import ReasoningParser, ReasoningParserManager
49
from vllm.sampling_params import BeamSearchParams, SamplingParams
50
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
51
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
52
53
                                                truncate_tool_call_ids,
                                                validate_request_params)
54
from vllm.utils import as_list
55
56
57
58
59
60

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

61
62
63
64
    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
65
        models: OpenAIServingModels,
66
67
68
69
70
        response_role: str,
        *,
        request_logger: Optional[RequestLogger],
        chat_template: Optional[str],
        chat_template_content_format: ChatTemplateContentFormatOption,
71
        trust_request_chat_template: bool = False,
72
        return_tokens_as_token_ids: bool = False,
73
        reasoning_parser: str = "",
74
        enable_auto_tools: bool = False,
75
        exclude_tools_when_tool_choice_none: bool = False,
76
77
        tool_parser: Optional[str] = None,
        enable_prompt_tokens_details: bool = False,
78
        enable_force_include_usage: bool = False,
79
        enable_log_outputs: bool = False,
80
        log_error_stack: bool = False,
81
    ) -> None:
82
        super().__init__(engine_client=engine_client,
83
                         model_config=model_config,
84
                         models=models,
85
                         request_logger=request_logger,
86
                         return_tokens_as_token_ids=return_tokens_as_token_ids,
87
88
                         enable_force_include_usage=enable_force_include_usage,
                         log_error_stack=log_error_stack)
89

90
        self.response_role = response_role
91
92
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
93
        self.trust_request_chat_template = trust_request_chat_template
94
        self.enable_log_outputs = enable_log_outputs
95

96
97
98
99
100
101
102
103
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

104
105
        self.reasoning_parser: Optional[Callable[[AnyTokenizer],
                                                 ReasoningParser]] = None
106
        if reasoning_parser:
107
108
109
110
            try:
                self.reasoning_parser = (
                    ReasoningParserManager.get_reasoning_parser(
                        reasoning_parser))
111
                assert self.reasoning_parser is not None
112
            except Exception as e:
113
114
                raise TypeError(
                    f"{reasoning_parser=} has not been registered") from e
115
116
        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
117
            try:
118
119
120
121
122
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
123
124
125
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
126
                raise TypeError("Error: --enable-auto-tool-choice requires "
127
128
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
129
130
        self.exclude_tools_when_tool_choice_none = (
            exclude_tools_when_tool_choice_none)
131

132
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
133
        self.enable_force_include_usage = enable_force_include_usage
134
135
136
        self.default_sampling_params = (
            self.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
137
138
139
140
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
            logger.info("Using default chat sampling params from %s: %s",
                        source, self.default_sampling_params)
141
142
143
144
        if self.model_config.hf_config.model_type == 'kimi_k2':
            self.tool_call_id_type = 'kimi_k2'
        else:
            self.tool_call_id_type = 'random'
145

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
        self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
                get_stop_tokens_for_assistant_actions())

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

163
    async def create_chat_completion(
164
165
        self,
        request: ChatCompletionRequest,
166
167
168
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
169
170
        """
        Chat Completion API similar to OpenAI's API.
171

172
173
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
174
        Chat Completion API.
175
176
177
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
178
            logger.error("Error with model %s", error_check_ret)
179
180
            return error_check_ret

181
182
183
184
185
186
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

187
        try:
188
189
            lora_request = self._maybe_get_adapters(
                request, supports_default_mm_loras=True)
190

191
            model_name = self.models.model_name(lora_request)
192

193
            tokenizer = await self.engine_client.get_tokenizer()
194

195
196
            tool_parser = self.tool_parser

197
            if isinstance(tokenizer, MistralTokenizer):
198
199
200
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
201
                maybe_serialize_tool_calls(request)
202
                truncate_tool_call_ids(request)
203
                validate_request_params(request)
204

205
206
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
207
208
                    and not isinstance(tokenizer, MistralTokenizer)
                    and not self.use_harmony):
209
210
211
212
213
214
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
215

216
217
218
            if (request.tools is None
                    or (request.tool_choice == "none"
                        and self.exclude_tools_when_tool_choice_none)):
219
220
221
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
222

223
224
            if not self.use_harmony:
                # Common case.
225
226
227
228
229
230
231
232
233
234
                request_chat_template = request.chat_template
                chat_template_kwargs = request.chat_template_kwargs
                if not self.trust_request_chat_template and (
                        request_chat_template is not None or
                    (chat_template_kwargs and
                     chat_template_kwargs.get("chat_template") is not None)):
                    return self.create_error_response(
                        "Chat template is passed with request, but "
                        "--trust-request-chat-template is not set. "
                        "Refused request with untrusted chat template.")
235
236
237
238
239
240
241
242
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = await self._preprocess_chat(
                    request,
                    tokenizer,
                    request.messages,
243
                    chat_template=request_chat_template or self.chat_template,
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
                    chat_template_content_format=self.
                    chat_template_content_format,
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = self._make_request_with_harmony(request)
261
262
        except (ValueError, TypeError, RuntimeError,
                jinja2.TemplateError) as e:
263
            logger.exception("Error in preprocessing prompt inputs")
264
            return self.create_error_response(f"{e} {e.__cause__}")
265

266
267
        request_id = "chatcmpl-" \
                     f"{self._base_request_id(raw_request, request.request_id)}"
268
269
270
271
272

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

273
        # Schedule the request and get the result generator.
274
        generators: list[AsyncGenerator[RequestOutput, None]] = []
275
        try:
276
277
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
278
279
280
281
282
283
284
285
286
287

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
                    default_sampling_params=self.default_sampling_params)

288
289
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
290
                        max_tokens, self.default_sampling_params)
291
292
                else:
                    sampling_params = request.to_sampling_params(
293
                        max_tokens, self.model_config.logits_processor_pattern,
294
                        self.default_sampling_params)
295
296
297
298

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
299
                                 lora_request=lora_request)
300
301
302
303
304
305
306
307
308

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
309
                        lora_request=lora_request,
310
311
312
313
314
315
316
317
318
319
320
321
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
                    )

                generators.append(generator)
322
        except ValueError as e:
323
            # TODO: Use a vllm-specific Validation Error
324
325
            return self.create_error_response(str(e))

326
327
328
        assert len(generators) == 1
        result_generator, = generators

329
330
331
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
332
333
334
335
336
337
338
339
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
                enable_force_include_usage=self.enable_force_include_usage)
340

341
342
        try:
            return await self.chat_completion_full_generator(
343
344
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
345
346
347
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
348
349
350
351

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
352
        return request.messages[-1]["role"]
353

354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
    @staticmethod
    def _bracket_level(s: str, opening='{', closing='}') -> int:
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
    def _filter_delta_text(delta_text: str,
                           previous_text: str) -> tuple[str, bool]:
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
            if c == '{':
                bracket_level += 1
                passed_zero = bracket_level == 0
            elif c == '}':
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
                if c == ',':
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
397
        current_text: Optional[str],
398
399
        delta_text: str,
        function_name_returned: bool,
400
        tool_call_idx: Optional[int] = None
401
    ) -> tuple[Optional[DeltaMessage], bool]:
402
403
404
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
            logger.debug('not enough tokens to parse into JSON yet')
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
                delta_text, previous_text)
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
            if not finishes_previous_tool and ("name" not in current_tool_call
                                               or "parameters"
                                               not in current_tool_call):
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
                    param_match = re.search(r'.*"parameters":\s*(.*)',
433
                                            current_text, re.DOTALL)
434
435
436
437
438
439
440
441
442
443
444
445
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
                        arguments, previous_text)

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
                    if (finishes_previous_tool
                            and "parameters" not in current_tool_call):
                        current_tool_call = obj[-2]

                    function_name_returned = True
446
447
448
449
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
                        idx=tool_call_idx)
450
                    delta_message = DeltaMessage(tool_calls=[
451
                        DeltaToolCall(id=tool_call_id,
452
453
454
                                      function=DeltaFunctionCall(
                                          name=current_tool_call["name"],
                                          arguments=arguments),
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
                                      index=len(obj) - 1,
                                      type="function")
                    ])

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
                        delta_text, previous_text)

                    if delta_text != "":
                        delta_message = DeltaMessage(tool_calls=[
                            DeltaToolCall(
                                function=DeltaFunctionCall(
                                    # OpenAI API returns None
                                    # instead of name every time
                                    name=None,
                                    arguments=delta_text),
471
                                index=len(obj) - 1)
472
473
474
475
476
477
                        ])
                    else:
                        delta_message = None

        return delta_message, function_name_returned

478
    async def chat_completion_stream_generator(
479
480
481
482
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
483
        model_name: str,
484
        conversation: list[ConversationMessage],
485
        tokenizer: AnyTokenizer,
486
        request_metadata: RequestResponseMetadata,
487
        enable_force_include_usage: bool,
488
    ) -> AsyncGenerator[str, None]:
489
        created_time = int(time.time())
490
        chunk_object_type: Final = "chat.completion.chunk"
491
        first_iteration = True
492
493

        # Send response for each token for each request.n (index)
494
495
496
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
497
        num_prompt_tokens = 0
498
        num_cached_tokens = None
499
500
501
502
503
        if self.use_harmony:
            harmony_parsers = [
                get_streamable_parser_for_assistant()
                for _ in range(num_choices)
            ]
504
505
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
506
507
508
509
510
511
512
513
514
515
516

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

517
        all_previous_token_ids: Optional[list[list[int]]]
518
        function_name_returned = [False] * num_choices
519
520
521
522
        if self.tool_call_id_type == 'kimi_k2':
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
523

524
525
526
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

527
528
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
529
        if tool_choice_auto or self.reasoning_parser:
530
531
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
532
533
534
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
535
536
        elif request.tool_choice == "required":
            all_previous_token_ids = None
537
        else:
538
            all_previous_token_ids = None
539

540
        try:
541
            if self.reasoning_parser:
542
543
544
545
546
547
548
                reasoning_parser = self.reasoning_parser(tokenizer)
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
549
550
551
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
552
                tool_parsers: list[Optional[ToolParser]] = [
553
554
555
556
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
557
        except Exception as e:
558
            logger.exception("Error in tool parser creation.")
559
560
561
562
563
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

564
565
        stream_options = request.stream_options
        if stream_options:
566
567
            include_usage = stream_options.include_usage \
                            or enable_force_include_usage
568
569
570
571
572
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

573
574
        try:
            async for res in result_generator:
575
576
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
577
578
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
579

580
581
582
583
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
584
                    num_cached_tokens = res.num_cached_tokens
585
586
                    # Send first response for each request.n (index) with
                    # the role
587
                    role = self.get_chat_request_role(request)
588
589
590

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
591
                    for i in range(num_choices):
592
593
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
594
595
596
597
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
598
599
                            logprobs=None,
                            finish_reason=None)
600
601

                        # return prompt_token_ids at the first chunk ever
602
603
604
605
606
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
607
608
609
610
                            model=model_name,
                            prompt_token_ids=(res.prompt_token_ids
                                              if request.return_token_ids else
                                              None))
611

612
613
614
615
616
617
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
618

619
620
621
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

622
623
                    # Send response to echo the input portion of the
                    # last message
624
                    if request.echo:
625
                        last_msg_content: Union[str, list[dict[str, str]]] = ""
626
627
628
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
629
630

                        if last_msg_content:
631
                            for i in range(num_choices):
632
633
634
635
636
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
637
                                        logprobs=None,
638
                                        finish_reason=None))
639
640
641
642
643
644
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
645
646
647
648
649
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
650

651
652
653
654
655
656
657
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
658
                    tool_parser = tool_parsers[i]
659
660
661
662

                    if finish_reason_sent[i]:
                        continue

663
                    if request.logprobs and request.top_logprobs is not None:
664
                        assert output.logprobs is not None, (
665
                            "Did not output logprobs")
666
                        logprobs = self._create_chat_logprobs(
667
668
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
669
                            tokenizer=tokenizer,
670
                            num_output_top_logprobs=request.top_logprobs,
671
672
                            return_as_token_id=request.
                            return_tokens_as_token_ids,
673
674
675
676
                        )
                    else:
                        logprobs = None

677
678
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
679
                        prev_recipient = harmony_parser.current_recipient
680
681
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
682
683
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
684
685
686
                        delta_text = harmony_parser.last_content_delta or ""
                    else:
                        delta_text = output.text
687
688
689
690
691
692

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

693
                    delta_message: Optional[DeltaMessage]
694

695
                    # just update previous_texts and previous_token_ids
696
                    if tool_choice_auto or self.reasoning_parser:
697
698
699
700
701
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
702
703
                        # avoid the None + list error.
                        if previous_token_ids:
704
                            current_token_ids = previous_token_ids + as_list(
705
706
                                output.token_ids)
                        else:
707
                            current_token_ids = as_list(output.token_ids)
708

709
                    if self.use_harmony:
710
                        if cur_channel == "final":
711
                            delta_message = DeltaMessage(content=delta_text)
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
                        elif cur_channel == "analysis":
                            if request.include_reasoning:
                                delta_message = DeltaMessage(
                                    reasoning_content=delta_text)
                            else:
                                delta_message = None
                        elif (cur_channel == "commentary" and cur_recipient
                              and cur_recipient.startswith("functions.")):
                            # Count completed tool calls to determine index
                            base_index = 0
                            for msg in harmony_parser.messages:
                                if (msg.channel == "commentary"
                                        and msg.recipient
                                        and msg.recipient.startswith(
                                            "functions.")):
                                    base_index += 1

                            if prev_recipient != cur_recipient:
                                tool_name = cur_recipient.split(
                                    "functions.", 1)[1]
                                delta_message = DeltaMessage(tool_calls=[
                                    DeltaToolCall(
                                        id=make_tool_call_id(),
                                        type="function",
                                        function=DeltaFunctionCall(
                                            name=tool_name,
                                            arguments="",
                                        ),
                                        index=base_index,
                                    )
                                ])
                            elif delta_text:
                                delta_message = DeltaMessage(tool_calls=[
                                    DeltaToolCall(
                                        index=base_index,
                                        function=DeltaFunctionCall(
                                            arguments=delta_text),
                                    )
                                ])
                            else:
                                delta_message = None

                            if delta_message is not None:
                                harmony_tools_streamed[i] = True
                        else:
                            delta_message = None
758
                    # handle streaming deltas for tools with named tool_choice
759
                    elif tool_choice_function_name:
760
                        if (self.reasoning_parser and not reasoning_end_arr[i]
761
762
763
764
765
766
767
768
769
770
771
772
773
                                and not reasoning_parser.is_reasoning_end(
                                    previous_token_ids)):
                            assert reasoning_parser is not None
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))
774
775
776
777
778
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Only keep 'content', remove 'reasoning_content'.
779
                            if reasoning_parser.is_reasoning_end(
780
781
782
783
                                    as_list(output.token_ids)) or (
                                        res.prompt_token_ids
                                        and reasoning_parser.is_reasoning_end(
                                            res.prompt_token_ids)):
784
                                reasoning_end_arr[i] = True
785
786
787
788
789
790
791
792
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
793
                            if self.reasoning_parser:
794
795
796
                                delta_text = previous_text + delta_text
                                current_text = ""

797
798
799
800
801
802
803
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        arguments=delta_text),
                                    index=i)
                            else:
                                delta_tool_call = DeltaToolCall(
804
                                    id=make_tool_call_id(),
805
806
807
808
809
810
811
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
                                        arguments=delta_text),
                                    index=i)
                                function_name_returned[i] = True

812
                            delta_message = DeltaMessage(tool_calls=[
813
                                delta_tool_call,
814
                            ])
815
                            tools_streamed[i] = True
816

817
818
819
820
821
822
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]

823
824
825
826
827
828
829
830
                        if self.reasoning_parser:
                            _, content = \
                                reasoning_parser.extract_reasoning_content(
                                    current_text,
                                    request
                                )
                        else:
                            content = current_text
831
832
833
                        delta_message, function_name_returned[i] = (
                            self.extract_tool_call_required_streaming(
                                previous_text=previous_text,
834
                                current_text=content,
835
                                delta_text=delta_text,
836
837
838
839
840
                                function_name_returned=fn_name_returned,
                                tool_call_idx=history_tool_call_cnt))
                        if (delta_message and delta_message.tool_calls and
                                delta_message.tool_calls[0].id is not None):
                            history_tool_call_cnt += 1
841
                            tools_streamed[i] = True
842

843
844
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
845
                    elif tool_choice_auto and self.reasoning_parser:
846
847
848
849
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
850
                        output_token_ids = as_list(output.token_ids)
851
852
853
854
855
856
857
858
859
                        if not reasoning_end_arr[i]:
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
860
                                    output_token_ids,
861
                                ))
862
863
864
865
866
867
868
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if res.prompt_token_ids and \
                                reasoning_parser.is_reasoning_end(
869
                                    res.prompt_token_ids):
870
                                reasoning_end_arr[i] = True
871
                                current_token_ids = output_token_ids
872
873
874
875
876
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
877
878
879
880
881
                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if reasoning_parser.is_reasoning_end(
882
                                    output_token_ids):
883
884
885
                                reasoning_end_arr[i] = True
                                current_token_ids =  \
                                    reasoning_parser.extract_content_ids(
886
                                        output_token_ids)
887
888
889
890
891
892
893
894
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
895
                            delta_token_ids = output_token_ids
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

                            delta_message = (
                                tool_parser.extract_tool_calls_streaming(
                                    previous_text=previous_text,
                                    current_text=current_text,
                                    delta_text=delta_text,
                                    previous_token_ids=previous_token_ids,
                                    current_token_ids=current_token_ids,
                                    delta_token_ids=delta_token_ids,
                                    request=request))
915
916
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
917
918
919
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
920
921
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
922
923
                                previous_text=previous_text,
                                current_text=current_text,
924
                                delta_text=delta_text,
925
926
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
927
928
                                delta_token_ids=output.token_ids,
                                request=request))
929
930
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
931

932
                    # when only reasoning
933
                    elif self.reasoning_parser:
934
935
936
937
938
939
940
941
942
                        delta_message = (reasoning_parser.
                                         extract_reasoning_content_streaming(
                                             previous_text,
                                             current_text,
                                             delta_text,
                                             previous_token_ids,
                                             current_token_ids,
                                             output.token_ids,
                                         ))
943
                    # handle streaming just a content delta
944
945
946
                    else:
                        delta_message = DeltaMessage(content=delta_text)

947
                    # update the previous values for the next iteration
948
949
                    if ((tool_choice_auto or self.reasoning_parser)
                            and not self.use_harmony):
950
951
952
953
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
954
955
956
957
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
958

959
                    # set the previous values for the next iteration
960
                    previous_num_tokens[i] += len(output.token_ids)
961
962
963
964
965
966

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
967
968
969
970
                        if output.finish_reason is None:
                            continue
                        else:
                            delta_message = DeltaMessage()
971

972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
                                if tc.function and tc.function.arguments)

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
987
                                output_token_ids=as_list(output.token_ids),
988
989
990
991
992
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

993
994
995
996
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
997
                            delta=delta_message,
998
                            logprobs=logprobs,
999
1000
1001
                            finish_reason=None,
                            token_ids=(as_list(output.token_ids)
                                       if request.return_token_ids else None))
1002
1003

                    # if the model is finished generating
1004
                    else:
1005
1006
1007
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1008
                        # only happens if we are NOT using structured outputs
1009
                        auto_tools_called = False
1010
                        if tool_parser:
1011
1012
1013
1014
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
1015
1016
1017
1018
1019
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
                            latest_delta_len = 0
                            if ((isinstance(
                                    delta_message.tool_calls[0].function,
                                    DeltaFunctionCall)) and isinstance(
                                        delta_message.tool_calls[0].function.
                                        arguments, str)):
                                latest_delta_len = len(
                                    delta_message.tool_calls[0].function.
                                    arguments)

1030
1031
1032
1033
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1034
1035
                                    "arguments", {}),
                                ensure_ascii=False)
1036

1037
                            # get what we've streamed so far for arguments
1038
1039
1040
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]
1041
1042
                            if (latest_delta_len > 0):
                                actual_call = actual_call[:-latest_delta_len]
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)
                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

1055
                        # Send the finish response for each request.n only once
1056
1057
1058
1059
1060
1061
1062
                        if auto_tools_called or tools_streamed[i] or (
                                self.use_harmony
                                and harmony_tools_streamed[i]):
                            finish_reason_ = "tool_calls"
                        else:
                            finish_reason_ = output.finish_reason \
                                if output.finish_reason else "stop"
1063
1064
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1065
                            delta=delta_message,
1066
                            logprobs=logprobs,
1067
                            finish_reason=finish_reason_,
1068
1069
1070
                            stop_reason=output.stop_reason,
                            token_ids=(as_list(output.token_ids)
                                       if request.return_token_ids else None))
1071

1072
                        finish_reason_sent[i] = True
1073

1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1090
                    data = chunk.model_dump_json(exclude_unset=True)
1091
1092
                    yield f"data: {data}\n\n"

1093
1094
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1095
1096
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1097
1098
1099
1100
1101
1102
1103
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
1115

1116
1117
1118
1119
1120
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
                        if previous_texts and i < len(previous_texts) else
                        f"<streaming_complete: {previous_num_tokens[i]} tokens>"
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
                        output_token_ids=
                        None,  # Consider also logging all token IDs
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1142

1143
        except Exception as e:
1144
            # TODO: Use a vllm-specific Validation Error
1145
            logger.exception("Error in chat completion stream generator.")
1146
1147
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1148
1149
1150
1151
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1152
1153
1154
1155
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1156
        model_name: str,
1157
        conversation: list[ConversationMessage],
1158
        tokenizer: AnyTokenizer,
1159
        request_metadata: RequestResponseMetadata,
1160
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
1161

1162
        created_time = int(time.time())
1163
        final_res: Optional[RequestOutput] = None
1164

1165
1166
1167
1168
1169
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1170
1171
1172
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1173

1174
1175
        assert final_res is not None

1176
        choices: list[ChatCompletionResponseChoice] = []
1177
1178
1179
1180
        if self.tool_call_id_type == 'kimi_k2':
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1181

1182
1183
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1184
            token_ids = output.token_ids
1185
            out_logprobs = output.logprobs
1186
            tool_call_info = None
1187

1188
1189
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1190
                logprobs = self._create_chat_logprobs(
1191
                    token_ids=token_ids,
1192
                    top_logprobs=out_logprobs,
1193
                    num_output_top_logprobs=request.top_logprobs,
1194
                    tokenizer=tokenizer,
1195
                    return_as_token_id=request.return_tokens_as_token_ids,
1196
1197
1198
                )
            else:
                logprobs = None
1199
1200

            if self.use_harmony:
1201
1202
1203
1204
                reasoning_content, content, _ = parse_chat_output(token_ids)
                if not request.include_reasoning:
                    reasoning_content = None

1205
1206
1207
1208
1209
1210
1211
1212
                if self.tool_parser is not None:
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1213
                    content = tool_call_info.content
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                    )
1226
1227
1228
1229
1230

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1231
1232
1233
                    finish_reason="tool_calls" if
                    (tool_call_info is not None
                     and tool_call_info.tools_called) else
1234
1235
1236
1237
1238
                    output.finish_reason if output.finish_reason else "stop",
                    stop_reason=output.stop_reason,
                )
                choices.append(choice_data)
                continue
1239

1240
            if self.reasoning_parser:
1241
1242
1243
1244
1245
                try:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1246
1247
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1248
1249
1250
                reasoning_content, content = (
                    reasoning_parser.extract_reasoning_content(
                        output.text, request=request))
1251
1252
                if not request.include_reasoning:
                    reasoning_content = None
1253
1254
1255
            else:
                reasoning_content = None
                content = output.text
1256

1257
            auto_tools_called = False
1258
1259
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1260
1261
1262
1263
            if (not self.enable_auto_tools or not self.tool_parser) and \
                (not isinstance(request.tool_choice,
                                ChatCompletionNamedToolChoiceParam
                                ) and request.tool_choice != "required"):
1264
1265
1266
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1267
1268
1269

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
1270
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
1271

1272
1273
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall
1274
1275
                message = ChatMessage(
                    role=role,
1276
                    reasoning_content=reasoning_content,
1277
1278
                    content="",
                    tool_calls=[
1279
                        tool_call_class(function=FunctionCall(
1280
                            name=request.tool_choice.function.name,
1281
1282
1283
1284
                            arguments=content,
                        ))
                    ],
                )
1285

1286
1287
1288
1289
1290
1291
            elif request.tool_choice and request.tool_choice == "required":
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall

                # the fields of FunctionDefinition are a superset of the
                # tool call outputs and can be used for parsing
1292
                assert content is not None
1293
                tool_calls = TypeAdapter(
1294
                    list[FunctionDefinition]).validate_json(content)
1295
1296
1297
1298
1299
1300
1301
                tool_call_ids = []
                for tool_call in tool_calls:
                    tool_call_ids.append(
                        make_tool_call_id(id_type=self.tool_call_id_type,
                                          func_name=tool_call.name,
                                          idx=history_tool_call_cnt))
                    history_tool_call_cnt += 1
1302
1303
1304
1305
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
1306
1307
1308
1309
1310
1311
1312
1313
1314
                        tool_call_class(id=tool_call_ids[i],
                                        function=FunctionCall(
                                            name=tool_call.name,
                                            arguments=json.dumps(
                                                tool_call.parameters,
                                                ensure_ascii=False)))
                        for i, tool_call in enumerate(tool_calls)
                    ],
                    reasoning_content=reasoning_content)
1315

1316
1317
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1318
            elif not request.tool_choice or request.tool_choice == "none":
1319

1320
1321
1322
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1323
1324
1325
1326
1327
1328
1329

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

1330
1331
1332
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
1333
                    logger.exception("Error in tool parser creation.")
1334
1335
                    return self.create_error_response(str(e))

1336
                tool_call_info = tool_parser.extract_tool_calls(
1337
                    content if content is not None else "", request=request)
1338
1339
1340
1341
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
1342
1343
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
1344
                                          reasoning_content=reasoning_content,
1345
1346
1347
1348
1349
1350
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1351
1352
1353
1354
1355
1356
1357
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
                    if (tool_call_info.content
                            and len(tool_call_info.content) > 0):
                        ret_content = tool_call_info.content
1358
1359
                    message = ChatMessage(role=role,
                                          reasoning_content=reasoning_content,
1360
                                          content=ret_content)
1361
1362
1363
1364
1365
1366
1367

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
1368
1369
1370
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1371

1372
1373
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1374
                message=message,
1375
                logprobs=logprobs,
1376
                finish_reason="tool_calls" if auto_tools_called else
1377
                output.finish_reason if output.finish_reason else "stop",
1378
1379
1380
1381
                stop_reason=output.stop_reason,
                token_ids=(as_list(output.token_ids)
                           if request.return_token_ids else None),
            )
1382

1383
1384
            choices.append(choice_data)

1385
        if request.echo:
1386
            last_msg_content: Union[str, list[dict[str, str]]] = ""
1387
1388
            if (conversation and "content" in conversation[-1]
                    and conversation[-1].get("role") == role):
1389
                last_msg_content = conversation[-1]["content"] or ""
1390
1391
1392
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
1393
1394

            for choice in choices:
1395
1396
                full_message = last_msg_content + (choice.message.content
                                                   or "")
1397
1398
                choice.message.content = full_message

1399
        assert final_res.prompt_token_ids is not None
1400
        num_prompt_tokens = len(final_res.prompt_token_ids)
1401
1402
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1403
1404
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
1405
1406
1407
1408
1409
1410
1411
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
1412
1413
1414

        request_metadata.final_usage_info = usage

1415
1416
1417
1418
1419
1420
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1421
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1422
1423
            prompt_token_ids=(final_res.prompt_token_ids
                              if request.return_token_ids else None),
Robert Shaw's avatar
Robert Shaw committed
1424
            kv_transfer_params=final_res.kv_transfer_params,
1425
1426
        )

1427
1428
1429
1430
1431
1432
1433
1434
1435
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1436
1437
1438
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
                                tc.function, "arguments"):
1439
                            tool_call_descriptions.append(
1440
                                f"{tc.function.name}({tc.function.arguments})")
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
                        output_token_ids = final_res.outputs[
                            choice.index].token_ids

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1460
        return response
1461
1462

    def _get_top_logprobs(
1463
            self, logprobs: dict[int, Logprob], top_logprobs: Optional[int],
1464
1465
            tokenizer: AnyTokenizer,
            should_return_as_token_id: bool) -> list[ChatCompletionLogProb]:
1466
        return [
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
            ChatCompletionLogProb(
                token=(token := self._get_decoded_token(
                    p[1],
                    p[0],
                    tokenizer,
                    return_as_token_id=should_return_as_token_id,
                )),
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
            ) for i, p in enumerate(logprobs.items())
1477
1478
1479
1480
1481
1482
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1483
        top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
1484
        tokenizer: AnyTokenizer,
1485
        num_output_top_logprobs: Optional[int] = None,
1486
        return_as_token_id: Optional[bool] = None,
1487
1488
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1489
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1490

1491
1492
        should_return_as_token_id = return_as_token_id if \
            return_as_token_id is not None else self.return_tokens_as_token_ids
1493
1494
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1495
1496
            if step_top_logprobs is None or step_top_logprobs.get(
                    token_id) is None:
1497
                if should_return_as_token_id:
1498
                    token = f"token_id:{token_id}"
1499
1500
                else:
                    token = tokenizer.decode(token_id)
1501

1502
1503
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1504
                        token=token,
1505
1506
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
1507
            else:
1508
1509
1510
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1511
1512
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1513
                        token=self._get_decoded_token(
1514
1515
1516
                            step_token,
                            token_id,
                            tokenizer,
1517
                            should_return_as_token_id,
1518
1519
1520
1521
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
1522
                        top_logprobs=self._get_top_logprobs(
1523
1524
                            step_top_logprobs, num_output_top_logprobs,
                            tokenizer, should_return_as_token_id),
1525
                    ))
1526
1527

        return ChatCompletionLogProbs(content=logprobs_content)
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1557
1558
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
1559
1560
1561
1562
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
    ):
        messages: list[OpenAIMessage] = []

        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1578
1579
1580
            python_description=None,
            with_custom_tools=request.tools is not None
            )
1581
1582
1583
        messages.append(sys_msg)

        # Add developer message.
1584
        dev_msg = get_developer_message(tools=request.tools)
1585
1586
1587
1588
        messages.append(dev_msg)

        # Add user message.
        for chat_msg in request.messages:
1589
            messages.extend(parse_chat_input(chat_msg))
1590
1591
1592
1593

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1594
1595
1596
1597
1598

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1599
        return messages, [prompt_token_ids], [engine_prompt]