serving_chat.py 74.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Callable, Final, Optional, Union
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16
from pydantic import TypeAdapter
17

18
from vllm.config import ModelConfig
19
from vllm.engine.protocol import EngineClient
20
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
21
                                         ConversationMessage,
22
23
                                         get_history_tool_calls_cnt,
                                         make_tool_call_id)
24
25
26
27
from vllm.entrypoints.harmony_utils import (
    get_developer_message, get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant, get_system_message, parse_chat_input,
    parse_chat_output, render_for_completion)
28
from vllm.entrypoints.logger import RequestLogger
29
from vllm.entrypoints.openai.protocol import (
30
31
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
32
    ChatCompletionRequest, ChatCompletionResponse,
33
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
34
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
35
36
    DeltaToolCall, ErrorResponse, FunctionCall, FunctionDefinition,
    PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo)
37
38
from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
                                                    clamp_prompt_logprobs)
39
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
40
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
41
42
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
    MistralToolCall)
43
from vllm.entrypoints.utils import get_max_tokens
44
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
45
from vllm.logger import init_logger
46
from vllm.logprobs import Logprob
47
from vllm.outputs import CompletionOutput, RequestOutput
48
from vllm.reasoning import ReasoningParser, ReasoningParserManager
49
from vllm.sampling_params import BeamSearchParams, SamplingParams
50
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
51
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
52
53
                                                truncate_tool_call_ids,
                                                validate_request_params)
54
from vllm.utils import as_list
55
56
57
58
59
60

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

61
62
63
64
    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
65
        models: OpenAIServingModels,
66
67
68
69
70
        response_role: str,
        *,
        request_logger: Optional[RequestLogger],
        chat_template: Optional[str],
        chat_template_content_format: ChatTemplateContentFormatOption,
71
        trust_request_chat_template: bool = False,
72
        return_tokens_as_token_ids: bool = False,
73
        reasoning_parser: str = "",
74
        enable_auto_tools: bool = False,
75
        exclude_tools_when_tool_choice_none: bool = False,
76
77
        tool_parser: Optional[str] = None,
        enable_prompt_tokens_details: bool = False,
78
        enable_force_include_usage: bool = False,
79
        enable_log_outputs: bool = False,
80
        log_error_stack: bool = False,
81
    ) -> None:
82
        super().__init__(engine_client=engine_client,
83
                         model_config=model_config,
84
                         models=models,
85
                         request_logger=request_logger,
86
                         return_tokens_as_token_ids=return_tokens_as_token_ids,
87
88
                         enable_force_include_usage=enable_force_include_usage,
                         log_error_stack=log_error_stack)
89

90
        self.response_role = response_role
91
92
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
93
        self.trust_request_chat_template = trust_request_chat_template
94
        self.enable_log_outputs = enable_log_outputs
95

96
97
98
99
100
101
102
103
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

104
105
        self.reasoning_parser: Optional[Callable[[AnyTokenizer],
                                                 ReasoningParser]] = None
106
        if reasoning_parser:
107
108
109
110
            try:
                self.reasoning_parser = (
                    ReasoningParserManager.get_reasoning_parser(
                        reasoning_parser))
111
                assert self.reasoning_parser is not None
112
            except Exception as e:
113
114
                raise TypeError(
                    f"{reasoning_parser=} has not been registered") from e
115
116
        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
117
            try:
118
119
120
121
122
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
123
124
125
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
126
                raise TypeError("Error: --enable-auto-tool-choice requires "
127
128
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
129
130
        self.exclude_tools_when_tool_choice_none = (
            exclude_tools_when_tool_choice_none)
131

132
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
133
        self.enable_force_include_usage = enable_force_include_usage
134
135
136
        self.default_sampling_params = (
            self.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
137
138
139
140
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
            logger.info("Using default chat sampling params from %s: %s",
                        source, self.default_sampling_params)
141
142
143
144
        if self.model_config.hf_config.model_type == 'kimi_k2':
            self.tool_call_id_type = 'kimi_k2'
        else:
            self.tool_call_id_type = 'random'
145

146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
        self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
                get_stop_tokens_for_assistant_actions())

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

163
    async def create_chat_completion(
164
165
        self,
        request: ChatCompletionRequest,
166
167
168
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
169
170
        """
        Chat Completion API similar to OpenAI's API.
171

172
173
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
174
        Chat Completion API.
175
176
177
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
178
            logger.error("Error with model %s", error_check_ret)
179
180
            return error_check_ret

181
182
183
184
185
186
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

187
        try:
188
189
            lora_request = self._maybe_get_adapters(
                request, supports_default_mm_loras=True)
190

191
            model_name = self.models.model_name(lora_request)
192

193
            tokenizer = await self.engine_client.get_tokenizer()
194

195
196
            tool_parser = self.tool_parser

197
            if isinstance(tokenizer, MistralTokenizer):
198
199
200
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
201
                maybe_serialize_tool_calls(request)
202
                truncate_tool_call_ids(request)
203
                validate_request_params(request)
204

205
206
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
207
208
                    and not isinstance(tokenizer, MistralTokenizer)
                    and not self.use_harmony):
209
210
211
212
213
214
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
215

216
217
218
            if (request.tools is None
                    or (request.tool_choice == "none"
                        and self.exclude_tools_when_tool_choice_none)):
219
220
221
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
222

223
224
            if not self.use_harmony:
                # Common case.
225
226
227
228
229
230
231
232
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
                    trust_request_chat_template=self.
                    trust_request_chat_template,
                )
                if error_check_ret is not None:
                    return error_check_ret
233
234
235
236
237
238
239
240
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = await self._preprocess_chat(
                    request,
                    tokenizer,
                    request.messages,
241
                    chat_template=request.chat_template or self.chat_template,
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
                    chat_template_content_format=self.
                    chat_template_content_format,
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
                (
                    conversation,
                    request_prompts,
                    engine_prompts,
                ) = self._make_request_with_harmony(request)
259
260
        except (ValueError, TypeError, RuntimeError,
                jinja2.TemplateError) as e:
261
            logger.exception("Error in preprocessing prompt inputs")
262
            return self.create_error_response(f"{e} {e.__cause__}")
263

264
265
        request_id = "chatcmpl-" \
                     f"{self._base_request_id(raw_request, request.request_id)}"
266
267
268
269
270

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

271
        # Schedule the request and get the result generator.
272
        generators: list[AsyncGenerator[RequestOutput, None]] = []
273
        try:
274
            for i, engine_prompt in enumerate(engine_prompts):
275
276
                prompt_text, _, _ = (self._get_prompt_components(
                    request_prompts[i]))
277
278
279
280
281
282
283
284
285
286

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
                    default_sampling_params=self.default_sampling_params)

287
                sampling_params: Union[SamplingParams, BeamSearchParams]
288
289
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
290
                        max_tokens, self.default_sampling_params)
291
292
                else:
                    sampling_params = request.to_sampling_params(
293
                        max_tokens, self.model_config.logits_processor_pattern,
294
                        self.default_sampling_params)
295
296
297
298

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
299
                                 lora_request=lora_request)
300
301
302
303
304
305
306
307
308

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
309
                        lora_request=lora_request,
310
311
                    )
                else:
312
313
314
315
316
317
318
319
320
321
                    engine_request, tokenization_kwargs = (
                        await self._process_inputs(
                            request_id,
                            engine_prompt,
                            sampling_params,
                            lora_request=lora_request,
                            trace_headers=trace_headers,
                            priority=request.priority,
                        ))

322
                    generator = self.engine_client.generate(
323
                        engine_request,
324
325
326
327
328
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
329
330
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
331
332
333
                    )

                generators.append(generator)
334
        except ValueError as e:
335
            # TODO: Use a vllm-specific Validation Error
336
337
            return self.create_error_response(str(e))

338
339
340
        assert len(generators) == 1
        result_generator, = generators

341
342
343
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
344
345
346
347
348
349
350
351
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
                enable_force_include_usage=self.enable_force_include_usage)
352

353
354
        try:
            return await self.chat_completion_full_generator(
355
356
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
357
358
359
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
360
361
362
363

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
364
        return request.messages[-1]["role"]
365

366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
    @staticmethod
    def _bracket_level(s: str, opening='{', closing='}') -> int:
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
    def _filter_delta_text(delta_text: str,
                           previous_text: str) -> tuple[str, bool]:
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
            if c == '{':
                bracket_level += 1
                passed_zero = bracket_level == 0
            elif c == '}':
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
                if c == ',':
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
409
        current_text: Optional[str],
410
411
        delta_text: str,
        function_name_returned: bool,
412
        tool_call_idx: Optional[int] = None
413
    ) -> tuple[Optional[DeltaMessage], bool]:
414
415
416
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
            logger.debug('not enough tokens to parse into JSON yet')
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
                delta_text, previous_text)
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
            if not finishes_previous_tool and ("name" not in current_tool_call
                                               or "parameters"
                                               not in current_tool_call):
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
                    param_match = re.search(r'.*"parameters":\s*(.*)',
445
                                            current_text, re.DOTALL)
446
447
448
449
450
451
452
453
454
455
456
457
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
                        arguments, previous_text)

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
                    if (finishes_previous_tool
                            and "parameters" not in current_tool_call):
                        current_tool_call = obj[-2]

                    function_name_returned = True
458
459
460
461
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
                        idx=tool_call_idx)
462
                    delta_message = DeltaMessage(tool_calls=[
463
                        DeltaToolCall(id=tool_call_id,
464
465
466
                                      function=DeltaFunctionCall(
                                          name=current_tool_call["name"],
                                          arguments=arguments),
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
                                      index=len(obj) - 1,
                                      type="function")
                    ])

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
                        delta_text, previous_text)

                    if delta_text != "":
                        delta_message = DeltaMessage(tool_calls=[
                            DeltaToolCall(
                                function=DeltaFunctionCall(
                                    # OpenAI API returns None
                                    # instead of name every time
                                    name=None,
                                    arguments=delta_text),
483
                                index=len(obj) - 1)
484
485
486
487
488
489
                        ])
                    else:
                        delta_message = None

        return delta_message, function_name_returned

490
    async def chat_completion_stream_generator(
491
492
493
494
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
495
        model_name: str,
496
        conversation: list[ConversationMessage],
497
        tokenizer: AnyTokenizer,
498
        request_metadata: RequestResponseMetadata,
499
        enable_force_include_usage: bool,
500
    ) -> AsyncGenerator[str, None]:
501
        created_time = int(time.time())
502
        chunk_object_type: Final = "chat.completion.chunk"
503
        first_iteration = True
504
505

        # Send response for each token for each request.n (index)
506
507
508
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
509
        num_prompt_tokens = 0
510
        num_cached_tokens = None
511
512
513
514
515
        if self.use_harmony:
            harmony_parsers = [
                get_streamable_parser_for_assistant()
                for _ in range(num_choices)
            ]
516
517
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
518
519
520
521
522
523
524
525
526
527
528

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

529
        all_previous_token_ids: Optional[list[list[int]]]
530
        function_name_returned = [False] * num_choices
531
532
533
534
        if self.tool_call_id_type == 'kimi_k2':
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
535

536
537
538
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

539
540
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
541
        if tool_choice_auto or self.reasoning_parser:
542
543
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
544
545
546
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
547
548
        elif request.tool_choice == "required":
            all_previous_token_ids = None
549
        else:
550
            all_previous_token_ids = None
551

552
        try:
553
            if self.reasoning_parser:
554
555
556
557
558
559
560
                reasoning_parser = self.reasoning_parser(tokenizer)
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
561
562
563
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
564
                tool_parsers: list[Optional[ToolParser]] = [
565
566
567
568
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
569
        except Exception as e:
570
            logger.exception("Error in tool parser creation.")
571
572
573
574
575
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

576
577
        stream_options = request.stream_options
        if stream_options:
578
579
            include_usage = stream_options.include_usage \
                            or enable_force_include_usage
580
581
582
583
584
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

585
586
        try:
            async for res in result_generator:
587
588
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
589
590
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
591

592
593
594
595
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
596
                    num_cached_tokens = res.num_cached_tokens
597
598
                    # Send first response for each request.n (index) with
                    # the role
599
                    role = self.get_chat_request_role(request)
600
601
602

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
603
                    for i in range(num_choices):
604
605
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
606
607
608
609
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
610
611
                            logprobs=None,
                            finish_reason=None)
612
613

                        # return prompt_token_ids at the first chunk ever
614
615
616
617
618
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
619
620
621
622
                            model=model_name,
                            prompt_token_ids=(res.prompt_token_ids
                                              if request.return_token_ids else
                                              None))
623

624
625
626
627
628
629
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
630

631
632
633
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

634
635
                    # Send response to echo the input portion of the
                    # last message
636
                    if request.echo:
637
                        last_msg_content: Union[str, list[dict[str, str]]] = ""
638
639
640
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
641
642

                        if last_msg_content:
643
                            for i in range(num_choices):
644
645
646
647
648
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
649
                                        logprobs=None,
650
                                        finish_reason=None))
651
652
653
654
655
656
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
657
658
659
660
661
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
662

663
664
665
666
667
668
669
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
670
                    tool_parser = tool_parsers[i]
671
672
673
674

                    if finish_reason_sent[i]:
                        continue

675
                    if request.logprobs and request.top_logprobs is not None:
676
                        assert output.logprobs is not None, (
677
                            "Did not output logprobs")
678
                        logprobs = self._create_chat_logprobs(
679
680
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
681
                            tokenizer=tokenizer,
682
                            num_output_top_logprobs=request.top_logprobs,
683
684
                            return_as_token_id=request.
                            return_tokens_as_token_ids,
685
686
687
688
                        )
                    else:
                        logprobs = None

689
690
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
691
                        prev_recipient = harmony_parser.current_recipient
692
                        delta_text = ""
693
694
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
695
696
                            delta_text += (harmony_parser.last_content_delta
                                           or "")
697
698
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
699
700
                    else:
                        delta_text = output.text
701
702
703
704
705
706

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

707
                    delta_message: Optional[DeltaMessage]
708

709
                    # just update previous_texts and previous_token_ids
710
                    if tool_choice_auto or self.reasoning_parser:
711
712
713
714
715
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
716
717
                        # avoid the None + list error.
                        if previous_token_ids:
718
                            current_token_ids = previous_token_ids + as_list(
719
720
                                output.token_ids)
                        else:
721
                            current_token_ids = as_list(output.token_ids)
722

723
                    if self.use_harmony:
724
                        if cur_channel == "final":
725
                            delta_message = DeltaMessage(content=delta_text)
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
                        elif cur_channel == "analysis":
                            if request.include_reasoning:
                                delta_message = DeltaMessage(
                                    reasoning_content=delta_text)
                            else:
                                delta_message = None
                        elif (cur_channel == "commentary" and cur_recipient
                              and cur_recipient.startswith("functions.")):
                            # Count completed tool calls to determine index
                            base_index = 0
                            for msg in harmony_parser.messages:
                                if (msg.channel == "commentary"
                                        and msg.recipient
                                        and msg.recipient.startswith(
                                            "functions.")):
                                    base_index += 1

                            if prev_recipient != cur_recipient:
                                tool_name = cur_recipient.split(
                                    "functions.", 1)[1]
                                delta_message = DeltaMessage(tool_calls=[
                                    DeltaToolCall(
                                        id=make_tool_call_id(),
                                        type="function",
                                        function=DeltaFunctionCall(
                                            name=tool_name,
                                            arguments="",
                                        ),
                                        index=base_index,
                                    )
                                ])
                            elif delta_text:
                                delta_message = DeltaMessage(tool_calls=[
                                    DeltaToolCall(
                                        index=base_index,
                                        function=DeltaFunctionCall(
                                            arguments=delta_text),
                                    )
                                ])
                            else:
                                delta_message = None

                            if delta_message is not None:
                                harmony_tools_streamed[i] = True
                        else:
                            delta_message = None
772
                    # handle streaming deltas for tools with named tool_choice
773
                    elif tool_choice_function_name:
774
                        if (self.reasoning_parser and not reasoning_end_arr[i]
775
776
777
778
779
780
781
782
783
784
785
786
787
                                and not reasoning_parser.is_reasoning_end(
                                    previous_token_ids)):
                            assert reasoning_parser is not None
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))
788
789
790
791
792
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Only keep 'content', remove 'reasoning_content'.
793
                            if reasoning_parser.is_reasoning_end(
794
795
796
797
                                    as_list(output.token_ids)) or (
                                        res.prompt_token_ids
                                        and reasoning_parser.is_reasoning_end(
                                            res.prompt_token_ids)):
798
                                reasoning_end_arr[i] = True
799
800
801
802
803
804
805
806
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
807
                            if self.reasoning_parser:
808
809
810
                                delta_text = previous_text + delta_text
                                current_text = ""

811
812
813
814
815
816
817
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        arguments=delta_text),
                                    index=i)
                            else:
                                delta_tool_call = DeltaToolCall(
818
                                    id=make_tool_call_id(),
819
820
821
822
823
824
825
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
                                        arguments=delta_text),
                                    index=i)
                                function_name_returned[i] = True

826
                            delta_message = DeltaMessage(tool_calls=[
827
                                delta_tool_call,
828
                            ])
829
                            tools_streamed[i] = True
830

831
832
833
834
835
836
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]

837
838
839
840
841
842
843
844
                        if self.reasoning_parser:
                            _, content = \
                                reasoning_parser.extract_reasoning_content(
                                    current_text,
                                    request
                                )
                        else:
                            content = current_text
845
846
847
                        delta_message, function_name_returned[i] = (
                            self.extract_tool_call_required_streaming(
                                previous_text=previous_text,
848
                                current_text=content,
849
                                delta_text=delta_text,
850
851
852
853
854
                                function_name_returned=fn_name_returned,
                                tool_call_idx=history_tool_call_cnt))
                        if (delta_message and delta_message.tool_calls and
                                delta_message.tool_calls[0].id is not None):
                            history_tool_call_cnt += 1
855
                            tools_streamed[i] = True
856

857
858
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
859
                    elif tool_choice_auto and self.reasoning_parser:
860
861
862
863
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
864
                        output_token_ids = as_list(output.token_ids)
865
866
867
868
869
870
871
872
873
                        if not reasoning_end_arr[i]:
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
874
                                    output_token_ids,
875
                                ))
876
877
878
879
880
881
882
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if res.prompt_token_ids and \
                                reasoning_parser.is_reasoning_end(
883
                                    res.prompt_token_ids):
884
                                reasoning_end_arr[i] = True
885
                                current_token_ids = output_token_ids
886
887
888
889
890
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
891
892
893
894
895
                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if reasoning_parser.is_reasoning_end(
896
                                    output_token_ids):
897
898
899
                                reasoning_end_arr[i] = True
                                current_token_ids =  \
                                    reasoning_parser.extract_content_ids(
900
                                        output_token_ids)
901
902
903
904
905
906
907
908
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
909
                            delta_token_ids = output_token_ids
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

                            delta_message = (
                                tool_parser.extract_tool_calls_streaming(
                                    previous_text=previous_text,
                                    current_text=current_text,
                                    delta_text=delta_text,
                                    previous_token_ids=previous_token_ids,
                                    current_token_ids=current_token_ids,
                                    delta_token_ids=delta_token_ids,
                                    request=request))
929
930
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
931
932
933
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
934
935
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
936
937
                                previous_text=previous_text,
                                current_text=current_text,
938
                                delta_text=delta_text,
939
940
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
941
942
                                delta_token_ids=output.token_ids,
                                request=request))
943
944
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
945

946
                    # when only reasoning
947
                    elif self.reasoning_parser:
948
949
950
951
952
953
954
955
956
                        delta_message = (reasoning_parser.
                                         extract_reasoning_content_streaming(
                                             previous_text,
                                             current_text,
                                             delta_text,
                                             previous_token_ids,
                                             current_token_ids,
                                             output.token_ids,
                                         ))
957
                    # handle streaming just a content delta
958
959
960
                    else:
                        delta_message = DeltaMessage(content=delta_text)

961
                    # update the previous values for the next iteration
962
963
                    if ((tool_choice_auto or self.reasoning_parser)
                            and not self.use_harmony):
964
965
966
967
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
968
969
970
971
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
972

973
                    # set the previous values for the next iteration
974
                    previous_num_tokens[i] += len(output.token_ids)
975
976
977
978
979
980

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
981
982
983
984
                        if output.finish_reason is None:
                            continue
                        else:
                            delta_message = DeltaMessage()
985

986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
                                if tc.function and tc.function.arguments)

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1001
                                output_token_ids=as_list(output.token_ids),
1002
1003
1004
1005
1006
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1007
1008
1009
1010
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1011
                            delta=delta_message,
1012
                            logprobs=logprobs,
1013
1014
1015
                            finish_reason=None,
                            token_ids=(as_list(output.token_ids)
                                       if request.return_token_ids else None))
1016
1017

                    # if the model is finished generating
1018
                    else:
1019
1020
1021
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1022
                        # only happens if we are NOT using structured outputs
1023
                        auto_tools_called = False
1024
                        if tool_parser:
1025
1026
1027
1028
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
1029
1030
1031
1032
1033
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
                            latest_delta_len = 0
                            if ((isinstance(
                                    delta_message.tool_calls[0].function,
                                    DeltaFunctionCall)) and isinstance(
                                        delta_message.tool_calls[0].function.
                                        arguments, str)):
                                latest_delta_len = len(
                                    delta_message.tool_calls[0].function.
                                    arguments)

1044
1045
1046
1047
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1048
1049
                                    "arguments", {}),
                                ensure_ascii=False)
1050

1051
                            # get what we've streamed so far for arguments
1052
1053
1054
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]
1055
1056
                            if (latest_delta_len > 0):
                                actual_call = actual_call[:-latest_delta_len]
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)
                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

1069
                        # Send the finish response for each request.n only once
1070
1071
1072
1073
1074
1075
1076
                        if auto_tools_called or tools_streamed[i] or (
                                self.use_harmony
                                and harmony_tools_streamed[i]):
                            finish_reason_ = "tool_calls"
                        else:
                            finish_reason_ = output.finish_reason \
                                if output.finish_reason else "stop"
1077
1078
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1079
                            delta=delta_message,
1080
                            logprobs=logprobs,
1081
                            finish_reason=finish_reason_,
1082
1083
1084
                            stop_reason=output.stop_reason,
                            token_ids=(as_list(output.token_ids)
                                       if request.return_token_ids else None))
1085

1086
                        finish_reason_sent[i] = True
1087

1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1104
                    data = chunk.model_dump_json(exclude_unset=True)
1105
1106
                    yield f"data: {data}\n\n"

1107
1108
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1109
1110
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1111
1112
1113
1114
1115
1116
1117
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
1129

1130
1131
1132
1133
1134
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
                        if previous_texts and i < len(previous_texts) else
                        f"<streaming_complete: {previous_num_tokens[i]} tokens>"
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
                        output_token_ids=
                        None,  # Consider also logging all token IDs
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1156

1157
        except Exception as e:
1158
            # TODO: Use a vllm-specific Validation Error
1159
            logger.exception("Error in chat completion stream generator.")
1160
1161
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1162
1163
1164
1165
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1166
1167
1168
1169
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1170
        model_name: str,
1171
        conversation: list[ConversationMessage],
1172
        tokenizer: AnyTokenizer,
1173
        request_metadata: RequestResponseMetadata,
1174
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
1175

1176
        created_time = int(time.time())
1177
        final_res: Optional[RequestOutput] = None
1178

1179
1180
1181
1182
1183
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1184
1185
1186
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1187

1188
1189
        assert final_res is not None

1190
        choices: list[ChatCompletionResponseChoice] = []
1191
1192
1193
1194
        if self.tool_call_id_type == 'kimi_k2':
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1195

1196
1197
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1198
            token_ids = output.token_ids
1199
            out_logprobs = output.logprobs
1200
            tool_call_info = None
1201

1202
1203
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1204
                logprobs = self._create_chat_logprobs(
1205
                    token_ids=token_ids,
1206
                    top_logprobs=out_logprobs,
1207
                    num_output_top_logprobs=request.top_logprobs,
1208
                    tokenizer=tokenizer,
1209
                    return_as_token_id=request.return_tokens_as_token_ids,
1210
1211
1212
                )
            else:
                logprobs = None
1213
1214

            if self.use_harmony:
1215
1216
1217
1218
                reasoning_content, content, _ = parse_chat_output(token_ids)
                if not request.include_reasoning:
                    reasoning_content = None

1219
1220
1221
1222
1223
1224
1225
1226
                if self.tool_parser is not None:
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1227
                    content = tool_call_info.content
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
                        reasoning_content=reasoning_content,
                        content=content,
                    )
1240
1241
1242
1243
1244

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1245
1246
1247
                    finish_reason="tool_calls" if
                    (tool_call_info is not None
                     and tool_call_info.tools_called) else
1248
1249
1250
1251
1252
                    output.finish_reason if output.finish_reason else "stop",
                    stop_reason=output.stop_reason,
                )
                choices.append(choice_data)
                continue
1253

1254
            if self.reasoning_parser:
1255
1256
1257
1258
1259
                try:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1260
1261
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1262
1263
1264
                reasoning_content, content = (
                    reasoning_parser.extract_reasoning_content(
                        output.text, request=request))
1265
1266
                if not request.include_reasoning:
                    reasoning_content = None
1267
1268
1269
            else:
                reasoning_content = None
                content = output.text
1270

1271
            auto_tools_called = False
1272
1273
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1274
1275
1276
1277
            if (not self.enable_auto_tools or not self.tool_parser) and \
                (not isinstance(request.tool_choice,
                                ChatCompletionNamedToolChoiceParam
                                ) and request.tool_choice != "required"):
1278
1279
1280
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1281
1282
1283

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
1284
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
1285

1286
1287
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall
1288
1289
                message = ChatMessage(
                    role=role,
1290
                    reasoning_content=reasoning_content,
1291
1292
                    content="",
                    tool_calls=[
1293
                        tool_call_class(function=FunctionCall(
1294
                            name=request.tool_choice.function.name,
1295
1296
1297
1298
                            arguments=content,
                        ))
                    ],
                )
1299

1300
1301
1302
1303
1304
1305
            elif request.tool_choice and request.tool_choice == "required":
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall

                # the fields of FunctionDefinition are a superset of the
                # tool call outputs and can be used for parsing
1306
                assert content is not None
1307
                tool_calls = TypeAdapter(
1308
                    list[FunctionDefinition]).validate_json(content)
1309
1310
1311
1312
1313
1314
1315
                tool_call_ids = []
                for tool_call in tool_calls:
                    tool_call_ids.append(
                        make_tool_call_id(id_type=self.tool_call_id_type,
                                          func_name=tool_call.name,
                                          idx=history_tool_call_cnt))
                    history_tool_call_cnt += 1
1316
1317
1318
1319
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
1320
1321
1322
1323
1324
1325
1326
1327
1328
                        tool_call_class(id=tool_call_ids[i],
                                        function=FunctionCall(
                                            name=tool_call.name,
                                            arguments=json.dumps(
                                                tool_call.parameters,
                                                ensure_ascii=False)))
                        for i, tool_call in enumerate(tool_calls)
                    ],
                    reasoning_content=reasoning_content)
1329

1330
1331
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1332
            elif not request.tool_choice or request.tool_choice == "none":
1333

1334
1335
1336
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1337
1338
1339
1340
1341
1342
1343

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

1344
1345
1346
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
1347
                    logger.exception("Error in tool parser creation.")
1348
1349
                    return self.create_error_response(str(e))

1350
                tool_call_info = tool_parser.extract_tool_calls(
1351
                    content if content is not None else "", request=request)
1352
1353
1354
1355
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
1356
1357
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
1358
                                          reasoning_content=reasoning_content,
1359
1360
1361
1362
1363
1364
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1365
1366
1367
1368
1369
1370
1371
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
                    if (tool_call_info.content
                            and len(tool_call_info.content) > 0):
                        ret_content = tool_call_info.content
1372
1373
                    message = ChatMessage(role=role,
                                          reasoning_content=reasoning_content,
1374
                                          content=ret_content)
1375
1376
1377
1378
1379
1380
1381

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
1382
1383
1384
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1385

1386
1387
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1388
                message=message,
1389
                logprobs=logprobs,
1390
                finish_reason="tool_calls" if auto_tools_called else
1391
                output.finish_reason if output.finish_reason else "stop",
1392
1393
1394
1395
                stop_reason=output.stop_reason,
                token_ids=(as_list(output.token_ids)
                           if request.return_token_ids else None),
            )
1396

1397
1398
            choices.append(choice_data)

1399
        if request.echo:
1400
            last_msg_content: Union[str, list[dict[str, str]]] = ""
1401
1402
            if (conversation and "content" in conversation[-1]
                    and conversation[-1].get("role") == role):
1403
                last_msg_content = conversation[-1]["content"] or ""
1404
1405
1406
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
1407
1408

            for choice in choices:
1409
1410
                full_message = last_msg_content + (choice.message.content
                                                   or "")
1411
1412
                choice.message.content = full_message

1413
        assert final_res.prompt_token_ids is not None
1414
        num_prompt_tokens = len(final_res.prompt_token_ids)
1415
1416
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1417
1418
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
1419
1420
1421
1422
1423
1424
1425
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
1426
1427
1428

        request_metadata.final_usage_info = usage

1429
1430
1431
1432
1433
1434
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1435
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1436
1437
            prompt_token_ids=(final_res.prompt_token_ids
                              if request.return_token_ids else None),
Robert Shaw's avatar
Robert Shaw committed
1438
            kv_transfer_params=final_res.kv_transfer_params,
1439
1440
        )

1441
1442
1443
1444
1445
1446
1447
1448
1449
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1450
1451
1452
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
                                tc.function, "arguments"):
1453
                            tool_call_descriptions.append(
1454
                                f"{tc.function.name}({tc.function.arguments})")
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
                        output_token_ids = final_res.outputs[
                            choice.index].token_ids

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1474
        return response
1475
1476

    def _get_top_logprobs(
1477
            self, logprobs: dict[int, Logprob], top_logprobs: Optional[int],
1478
1479
            tokenizer: AnyTokenizer,
            should_return_as_token_id: bool) -> list[ChatCompletionLogProb]:
1480
        return [
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
            ChatCompletionLogProb(
                token=(token := self._get_decoded_token(
                    p[1],
                    p[0],
                    tokenizer,
                    return_as_token_id=should_return_as_token_id,
                )),
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
            ) for i, p in enumerate(logprobs.items())
1491
1492
1493
1494
1495
1496
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1497
        top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
1498
        tokenizer: AnyTokenizer,
1499
        num_output_top_logprobs: Optional[int] = None,
1500
        return_as_token_id: Optional[bool] = None,
1501
1502
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1503
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1504

1505
1506
        should_return_as_token_id = return_as_token_id if \
            return_as_token_id is not None else self.return_tokens_as_token_ids
1507
1508
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1509
1510
            if step_top_logprobs is None or step_top_logprobs.get(
                    token_id) is None:
1511
                if should_return_as_token_id:
1512
                    token = f"token_id:{token_id}"
1513
1514
                else:
                    token = tokenizer.decode(token_id)
1515

1516
1517
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1518
                        token=token,
1519
1520
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
1521
            else:
1522
1523
1524
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1525
1526
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1527
                        token=self._get_decoded_token(
1528
1529
1530
                            step_token,
                            token_id,
                            tokenizer,
1531
                            should_return_as_token_id,
1532
1533
1534
1535
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
1536
                        top_logprobs=self._get_top_logprobs(
1537
1538
                            step_top_logprobs, num_output_top_logprobs,
                            tokenizer, should_return_as_token_id),
1539
                    ))
1540
1541

        return ChatCompletionLogProbs(content=logprobs_content)
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1571
1572
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
1573
1574
1575
1576
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
    ):
        messages: list[OpenAIMessage] = []

        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1592
1593
1594
            python_description=None,
            with_custom_tools=request.tools is not None
            )
1595
1596
1597
        messages.append(sys_msg)

        # Add developer message.
1598
        dev_msg = get_developer_message(tools=request.tools)
1599
1600
1601
1602
        messages.append(dev_msg)

        # Add user message.
        for chat_msg in request.messages:
1603
            messages.extend(parse_chat_input(chat_msg))
1604
1605
1606
1607

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1608
1609
1610
1611
1612

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1613
        return messages, [prompt_token_ids], [engine_prompt]