serving_chat.py 81 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
9
from typing import Final
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from openai_harmony import Message as OpenAIMessage
16

17
from vllm.engine.protocol import EngineClient
18
19
20
21
22
23
from vllm.entrypoints.chat_utils import (
    ChatTemplateContentFormatOption,
    ConversationMessage,
    get_history_tool_calls_cnt,
    make_tool_call_id,
)
24
25
from vllm.entrypoints.logger import RequestLogger
from vllm.entrypoints.openai.parser.harmony_utils import (
26
27
28
29
    get_developer_message,
    get_stop_tokens_for_assistant_actions,
    get_streamable_parser_for_assistant,
    get_system_message,
30
    parse_chat_inputs_to_harmony_messages,
31
32
33
    parse_chat_output,
    render_for_completion,
)
34
from vllm.entrypoints.openai.protocol import (
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
    ChatCompletionLogProb,
    ChatCompletionLogProbs,
    ChatCompletionLogProbsContent,
    ChatCompletionNamedToolChoiceParam,
    ChatCompletionRequest,
    ChatCompletionResponse,
    ChatCompletionResponseChoice,
    ChatCompletionResponseStreamChoice,
    ChatCompletionStreamResponse,
    ChatMessage,
    DeltaFunctionCall,
    DeltaMessage,
    DeltaToolCall,
    ErrorResponse,
    PromptTokenUsageInfo,
    RequestResponseMetadata,
    ToolCall,
    UsageInfo,
)
54
55
56
57
58
from vllm.entrypoints.openai.serving_engine import (
    GenerationError,
    OpenAIServing,
    clamp_prompt_logprobs,
)
59
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
60
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
61
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
62
from vllm.inputs.data import TokensPrompt
63
from vllm.logger import init_logger
64
from vllm.logprobs import Logprob
65
from vllm.outputs import CompletionOutput, RequestOutput
66
from vllm.sampling_params import BeamSearchParams, SamplingParams
67
68
69
from vllm.tokenizers import TokenizerLike
from vllm.tokenizers.mistral import (
    MistralTokenizer,
70
71
72
73
    maybe_serialize_tool_calls,
    truncate_tool_call_ids,
    validate_request_params,
)
74
75
from vllm.tool_parsers import ToolParser
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
76
from vllm.utils.collection_utils import as_list
77
from vllm.v1.sample.logits_processor import validate_logits_processors_parameters
78
79
80
81
82

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):
83
84
85
    def __init__(
        self,
        engine_client: EngineClient,
86
        models: OpenAIServingModels,
87
88
        response_role: str,
        *,
89
90
        request_logger: RequestLogger | None,
        chat_template: str | None,
91
        chat_template_content_format: ChatTemplateContentFormatOption,
92
        trust_request_chat_template: bool = False,
93
        return_tokens_as_token_ids: bool = False,
94
        reasoning_parser: str = "",
95
        enable_auto_tools: bool = False,
96
        exclude_tools_when_tool_choice_none: bool = False,
97
        tool_parser: str | None = None,
98
        enable_prompt_tokens_details: bool = False,
99
        enable_force_include_usage: bool = False,
100
        enable_log_outputs: bool = False,
101
        log_error_stack: bool = False,
102
    ) -> None:
103
104
105
106
107
108
109
        super().__init__(
            engine_client=engine_client,
            models=models,
            request_logger=request_logger,
            return_tokens_as_token_ids=return_tokens_as_token_ids,
            log_error_stack=log_error_stack,
        )
110

111
        self.response_role = response_role
112
113
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
114
        self.trust_request_chat_template = trust_request_chat_template
115
        self.enable_log_outputs = enable_log_outputs
116

117
118
119
        # set up logits processors
        self.logits_processors = self.model_config.logits_processors

120
121
122
123
        # set up reasoning parser
        self.reasoning_parser = self._get_reasoning_parser(
            reasoning_parser_name=reasoning_parser
        )
124
125
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
126
127
        self.tool_parser = self._get_tool_parser(
            tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
128
129
        )
        self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
130

131
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
132
        self.enable_force_include_usage = enable_force_include_usage
133
        self.default_sampling_params = self.model_config.get_diff_sampling_param()
134
        if self.default_sampling_params:
135
136
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
137
138
139
140
141
142
143
            logger.info(
                "Using default chat sampling params from %s: %s",
                source,
                self.default_sampling_params,
            )
        if self.model_config.hf_config.model_type == "kimi_k2":
            self.tool_call_id_type = "kimi_k2"
144
        else:
145
            self.tool_call_id_type = "random"
146

147
        self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
148
149
150
151
        if self.use_harmony:
            if "stop_token_ids" not in self.default_sampling_params:
                self.default_sampling_params["stop_token_ids"] = []
            self.default_sampling_params["stop_token_ids"].extend(
152
153
                get_stop_tokens_for_assistant_actions()
            )
154
155
156
157
158
159
160
161
162
163
164

        # NOTE(woosuk): While OpenAI's chat completion API supports browsing
        # for some models, currently vLLM doesn't support it. Please use the
        # Responses API instead.
        self.supports_browsing = False
        self.browser_tool = None
        # NOTE(woosuk): Chat completion API does not support code interpreter.
        # Please use the Responses API instead.
        self.supports_code_interpreter = False
        self.python_tool = None

165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
    async def warmup(self) -> None:
        """
        Warm up the chat template processing to avoid first-request latency.

        This method triggers Jinja2 template compilation and content format
        detection that would otherwise happen on the first real request,
        causing increased latency on the first request.
        """
        logger.info("Warming up chat template processing...")
        start_time = time.perf_counter()

        try:
            # Get the tokenizer from the engine
            tokenizer = await self.engine_client.get_tokenizer()

            # Create a minimal dummy request
            dummy_request = ChatCompletionRequest(
                messages=[{"role": "user", "content": "warmup"}],
                model=None,
                max_completion_tokens=1,
            )

            # Call _preprocess_chat to trigger template compilation
            # This forces:
            # 1. Chat template content format detection
            # 2. Jinja2 template compilation
            # 3. Tokenizer initialization for chat
            await self._preprocess_chat(
                dummy_request,
                tokenizer,
                dummy_request.messages,
                chat_template=self.chat_template,
                chat_template_content_format=self.chat_template_content_format,
                add_generation_prompt=True,
                continue_final_message=False,
                tool_dicts=None,
                documents=None,
                chat_template_kwargs=None,
                tool_parser=None,
                add_special_tokens=False,
            )

            elapsed = (time.perf_counter() - start_time) * 1000
            logger.info("Chat template warmup completed in %.1fms", elapsed)

        except Exception:
            # Log but don't fail server startup if warmup fails
            logger.exception("Chat template warmup failed")

214
    async def create_chat_completion(
215
216
        self,
        request: ChatCompletionRequest,
217
218
        raw_request: Request | None = None,
    ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
219
220
        """
        Chat Completion API similar to OpenAI's API.
221

222
223
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
224
        Chat Completion API.
225
226
227
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
228
            logger.error("Error with model %s", error_check_ret)
229
230
            return error_check_ret

231
232
233
234
235
236
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

237
        try:
238
            lora_request = self._maybe_get_adapters(
239
240
                request, supports_default_mm_loras=True
            )
241

242
            model_name = self.models.model_name(lora_request)
243

244
            tokenizer = await self.engine_client.get_tokenizer()
245

246
247
            tool_parser = self.tool_parser

248
            if isinstance(tokenizer, MistralTokenizer):
249
250
251
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
252
                maybe_serialize_tool_calls(request)
253
                truncate_tool_call_ids(request)
254
                validate_request_params(request)
255

256
257
258
259
260
261
            if (
                request.tool_choice == "auto"
                and not (self.enable_auto_tools and tool_parser is not None)
                and not isinstance(tokenizer, MistralTokenizer)
                and not self.use_harmony
            ):
262
263
264
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
265
                    '"auto" tool choice requires '
266
267
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
268

269
270
271
272
            if request.tools is None or (
                request.tool_choice == "none"
                and self.exclude_tools_when_tool_choice_none
            ):
273
274
275
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
276

277
278
            if not self.use_harmony:
                # Common case.
279
280
281
                error_check_ret = self._validate_chat_template(
                    request_chat_template=request.chat_template,
                    chat_template_kwargs=request.chat_template_kwargs,
282
                    trust_request_chat_template=self.trust_request_chat_template,
283
284
285
                )
                if error_check_ret is not None:
                    return error_check_ret
286
                conversation, engine_prompts = await self._preprocess_chat(
287
288
289
                    request,
                    tokenizer,
                    request.messages,
290
                    chat_template=request.chat_template or self.chat_template,
291
                    chat_template_content_format=self.chat_template_content_format,
292
293
294
295
296
297
298
299
300
301
                    add_generation_prompt=request.add_generation_prompt,
                    continue_final_message=request.continue_final_message,
                    tool_dicts=tool_dicts,
                    documents=request.documents,
                    chat_template_kwargs=request.chat_template_kwargs,
                    tool_parser=tool_parser,
                    add_special_tokens=request.add_special_tokens,
                )
            else:
                # For GPT-OSS.
302
                conversation, engine_prompts = self._make_request_with_harmony(request)
303
        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
304
            logger.exception("Error in preprocessing prompt inputs")
305
            return self.create_error_response(f"{e} {e.__cause__}")
306

307
308
309
        request_id = (
            f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
        )
310
311
312
313
314

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

315
316
317
        # Extract data_parallel_rank from header (router can inject it)
        data_parallel_rank = self._get_data_parallel_rank(raw_request)

318
        # Schedule the request and get the result generator.
319
        generators: list[AsyncGenerator[RequestOutput, None]] = []
320
        try:
321
            for i, engine_prompt in enumerate(engine_prompts):
322
                prompt_text, _, _ = self._get_prompt_components(engine_prompt)
323
324
325
326
327
                # If we are creating sub requests for multiple prompts, ensure that they
                # have unique request ids.
                sub_request_id = (
                    request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                )
328
329
330
331
332
333
334
335

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
336
337
                    default_sampling_params=self.default_sampling_params,
                )
338

339
                sampling_params: SamplingParams | BeamSearchParams
340
341
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
342
343
                        max_tokens, self.default_sampling_params
                    )
344
345
                else:
                    sampling_params = request.to_sampling_params(
346
347
348
349
                        max_tokens,
                        self.model_config.logits_processor_pattern,
                        self.default_sampling_params,
                    )
350
351
352
353
                    validate_logits_processors_parameters(
                        self.logits_processors,
                        sampling_params,
                    )
354

355
                self._log_inputs(
356
                    sub_request_id,
357
                    engine_prompt,
358
359
360
                    params=sampling_params,
                    lora_request=lora_request,
                )
361

362
363
364
365
366
                trace_headers = (
                    None
                    if raw_request is None
                    else await self._get_trace_headers(raw_request.headers)
                )
367
368

                if isinstance(sampling_params, BeamSearchParams):
369
                    generator = self.beam_search(
370
                        prompt=engine_prompt,
371
                        request_id=sub_request_id,
372
                        params=sampling_params,
373
                        lora_request=lora_request,
374
                        trace_headers=trace_headers,
375
376
                    )
                else:
377
                    engine_request, tokenization_kwargs = await self._process_inputs(
378
                        sub_request_id,
379
380
381
382
383
384
                        engine_prompt,
                        sampling_params,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
                    )
385

386
                    generator = self.engine_client.generate(
387
                        engine_request,
388
                        sampling_params,
389
                        sub_request_id,
390
391
392
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
393
394
                        prompt_text=prompt_text,
                        tokenization_kwargs=tokenization_kwargs,
395
                        data_parallel_rank=data_parallel_rank,
396
397
398
                    )

                generators.append(generator)
399
        except ValueError as e:
400
            # TODO: Use a vllm-specific Validation Error
401
402
            return self.create_error_response(str(e))

403
        assert len(generators) == 1
404
        (result_generator,) = generators
405

406
407
408
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
409
410
411
412
413
414
415
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
416
            )
417

418
419
        try:
            return await self.chat_completion_full_generator(
420
421
422
423
424
425
426
427
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
            )
428
429
        except GenerationError as e:
            return self._convert_generation_error_to_response(e)
430
431
432
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
433
434
435
436

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
437
        return request.messages[-1]["role"]
438

439
    @staticmethod
440
    def _bracket_level(s: str, opening="{", closing="}") -> int:
441
442
443
444
445
446
447
448
449
450
451
452
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
453
    def _filter_delta_text(delta_text: str, previous_text: str) -> tuple[str, bool]:
454
455
456
457
458
459
460
461
462
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
463
            if c == "{":
464
465
                bracket_level += 1
                passed_zero = bracket_level == 0
466
            elif c == "}":
467
468
469
470
471
472
473
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
474
                if c == ",":
475
476
477
478
479
480
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
481
        current_text: str | None,
482
483
        delta_text: str,
        function_name_returned: bool,
484
485
        tool_call_idx: int | None = None,
    ) -> tuple[DeltaMessage | None, bool]:
486
487
488
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
489
490
491
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
492
            logger.debug("not enough tokens to parse into JSON yet")
493
494
495
496
497
498
499
500
501
502
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
503
504
                delta_text, previous_text
            )
505
506
507
508
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
509
510
511
            if not finishes_previous_tool and (
                "name" not in current_tool_call or "parameters" not in current_tool_call
            ):
512
513
514
515
516
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
517
518
519
                    param_match = re.search(
                        r'.*"parameters":\s*(.*)', current_text, re.DOTALL
                    )
520
521
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
522
523
                        arguments, previous_text
                    )
524
525
526
527

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
528
                    if finishes_previous_tool and "parameters" not in current_tool_call:
529
530
531
                        current_tool_call = obj[-2]

                    function_name_returned = True
532
533
534
                    tool_call_id = make_tool_call_id(
                        id_type=self.tool_call_id_type,
                        func_name=current_tool_call["name"],
535
536
537
538
539
540
541
542
543
544
545
546
547
548
                        idx=tool_call_idx,
                    )
                    delta_message = DeltaMessage(
                        tool_calls=[
                            DeltaToolCall(
                                id=tool_call_id,
                                function=DeltaFunctionCall(
                                    name=current_tool_call["name"], arguments=arguments
                                ),
                                index=len(obj) - 1,
                                type="function",
                            )
                        ]
                    )
549
550
551

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
552
553
                        delta_text, previous_text
                    )
554
555

                    if delta_text != "":
556
557
558
559
560
561
562
563
564
565
566
567
568
                        delta_message = DeltaMessage(
                            tool_calls=[
                                DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        # OpenAI API returns None
                                        # instead of name every time
                                        name=None,
                                        arguments=delta_text,
                                    ),
                                    index=len(obj) - 1,
                                )
                            ]
                        )
569
570
571
572
573
                    else:
                        delta_message = None

        return delta_message, function_name_returned

574
    async def chat_completion_stream_generator(
575
576
577
578
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
579
        model_name: str,
580
        conversation: list[ConversationMessage],
581
        tokenizer: TokenizerLike | None,
582
        request_metadata: RequestResponseMetadata,
583
    ) -> AsyncGenerator[str, None]:
584
        created_time = int(time.time())
585
        chunk_object_type: Final = "chat.completion.chunk"
586
        first_iteration = True
587
588

        # Send response for each token for each request.n (index)
589
590
591
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
592
        num_prompt_tokens = 0
593
        num_cached_tokens = None
594
595
        if self.use_harmony:
            harmony_parsers = [
596
                get_streamable_parser_for_assistant() for _ in range(num_choices)
597
            ]
598
599
            harmony_tools_streamed = [False] * num_choices
        tools_streamed = [False] * num_choices
600
601
602
603
604
605
606
607
608

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
609
610
            and self._should_stream_with_auto_tool_parsing(request)
        )
611

612
        all_previous_token_ids: list[list[int]] | None
613
        function_name_returned = [False] * num_choices
614
        if self.tool_call_id_type == "kimi_k2":
615
616
617
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
618

619
620
621
        # Always track previous_texts for comprehensive output logging
        previous_texts = [""] * num_choices

622
623
        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
624
        if tool_choice_auto or self.reasoning_parser:
625
626
            # These are only required in "auto" tool choice case
            all_previous_token_ids = [[]] * num_choices
627
628
629
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
630
        else:
631
            all_previous_token_ids = None
632

633
        try:
634
            if self.reasoning_parser:
635
636
637
638
639
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

640
641
642
643
                reasoning_parser = self.reasoning_parser(
                    tokenizer,
                    chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                )
644
645
646
647
648
649
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
650
651
652
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
653
654
655
656
657
                if tokenizer is None:
                    raise ValueError(
                        "Tokenizer not available when `skip_tokenizer_init=True`"
                    )

658
                tool_parsers: list[ToolParser | None] = [
659
660
661
662
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
663
        except Exception as e:
664
            logger.exception("Error in tool parser creation.")
665
666
667
668
669
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

670
        stream_options = request.stream_options
671
672
673
        include_usage, include_continuous_usage = should_include_usage(
            stream_options, self.enable_force_include_usage
        )
674

675
676
        try:
            async for res in result_generator:
677
678
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
679
680
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
681

682
683
684
685
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
686
                    num_cached_tokens = res.num_cached_tokens
687
688
                    # Send first response for each request.n (index) with
                    # the role
689
                    role = self.get_chat_request_role(request)
690
691
692

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
693
                    for i in range(num_choices):
694
695
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
696
697
698
699
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
700
                            logprobs=None,
701
702
                            finish_reason=None,
                        )
703
704

                        # return prompt_token_ids at the first chunk ever
705
706
707
708
709
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
710
                            model=model_name,
711
712
713
714
715
716
                            prompt_token_ids=(
                                res.prompt_token_ids
                                if request.return_token_ids
                                else None
                            ),
                        )
717

718
719
720
721
722
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
723
724
                                total_tokens=num_prompt_tokens,
                            )
725

726
727
728
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

729
730
                    # Send response to echo the input portion of the
                    # last message
731
                    if request.echo:
732
                        last_msg_content: str | list[dict[str, str]] = ""
733
734
735
736
737
                        if (
                            conversation
                            and "content" in conversation[-1]
                            and conversation[-1].get("role") == role
                        ):
738
                            last_msg_content = conversation[-1]["content"] or ""
739
740

                        if last_msg_content:
741
                            for i in range(num_choices):
742
743
744
745
746
747
                                choice_data = ChatCompletionResponseStreamChoice(
                                    index=i,
                                    delta=DeltaMessage(content=last_msg_content),
                                    logprobs=None,
                                    finish_reason=None,
                                )
748
749
750
751
752
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
753
754
                                    model=model_name,
                                )
755
756
757
758
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
759
760
                                        total_tokens=num_prompt_tokens,
                                    )
761

762
                                data = chunk.model_dump_json(exclude_unset=True)
763
764
765
766
767
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
768
                    tool_parser = tool_parsers[i]
769
770
771
772

                    if finish_reason_sent[i]:
                        continue

773
                    if request.logprobs and request.top_logprobs is not None:
774
                        assert output.logprobs is not None, "Did not output logprobs"
775
                        logprobs = self._create_chat_logprobs(
776
777
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
778
                            tokenizer=tokenizer,
779
                            num_output_top_logprobs=request.top_logprobs,
780
                            return_as_token_id=request.return_tokens_as_token_ids,
781
782
783
784
                        )
                    else:
                        logprobs = None

785
786
                    if self.use_harmony:
                        harmony_parser = harmony_parsers[i]
787
                        prev_recipient = harmony_parser.current_recipient
788
                        delta_text = ""
789
790
                        for token_id in output.token_ids:
                            harmony_parser.process(token_id)
791
                            delta_text += harmony_parser.last_content_delta or ""
792
793
                        cur_channel = harmony_parser.current_channel
                        cur_recipient = harmony_parser.current_recipient
794
795
                    else:
                        delta_text = output.text
796

797
798
799
800
801
                    if (
                        not delta_text
                        and not output.token_ids
                        and not previous_num_tokens[i]
                    ):
802
803
804
                        # Chunked prefill case, don't return empty chunks
                        continue

805
                    delta_message: DeltaMessage | None
806

807
                    # just update previous_texts and previous_token_ids
808
                    if tool_choice_auto or self.reasoning_parser:
809
810
811
812
813
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
814
815
                        # avoid the None + list error.
                        if previous_token_ids:
816
                            current_token_ids = previous_token_ids + as_list(
817
818
                                output.token_ids
                            )
819
                        else:
820
                            current_token_ids = as_list(output.token_ids)
821

822
                    if self.use_harmony:
823
                        if cur_channel == "final":
824
                            delta_message = DeltaMessage(content=delta_text)
825
826
                        elif cur_channel == "analysis":
                            if request.include_reasoning:
827
                                delta_message = DeltaMessage(reasoning=delta_text)
828
829
                            else:
                                delta_message = None
830
831
832
833
834
                        elif (
                            cur_channel == "commentary"
                            and cur_recipient
                            and cur_recipient.startswith("functions.")
                        ):
835
836
837
                            # Count completed tool calls to determine index
                            base_index = 0
                            for msg in harmony_parser.messages:
838
839
840
841
842
                                if (
                                    msg.channel == "commentary"
                                    and msg.recipient
                                    and msg.recipient.startswith("functions.")
                                ):
843
844
845
                                    base_index += 1

                            if prev_recipient != cur_recipient:
846
847
848
849
850
851
852
853
854
855
856
857
858
859
                                tool_name = cur_recipient.split("functions.", 1)[1]
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            id=make_tool_call_id(),
                                            type="function",
                                            function=DeltaFunctionCall(
                                                name=tool_name,
                                                arguments="",
                                            ),
                                            index=base_index,
                                        )
                                    ]
                                )
860
                            elif delta_text:
861
862
863
864
865
866
867
868
869
870
                                delta_message = DeltaMessage(
                                    tool_calls=[
                                        DeltaToolCall(
                                            index=base_index,
                                            function=DeltaFunctionCall(
                                                arguments=delta_text
                                            ),
                                        )
                                    ]
                                )
871
872
873
874
875
                            else:
                                delta_message = None

                            if delta_message is not None:
                                harmony_tools_streamed[i] = True
876
877
878
                        elif cur_channel == "commentary":
                            # Tool call preambles meant to be shown to the user
                            delta_message = DeltaMessage(content=delta_text)
879
880
                        else:
                            delta_message = None
881
                    # handle streaming deltas for tools with named tool_choice
882
                    elif tool_choice_function_name:
883
884
885
886
887
888
889
                        if (
                            self.reasoning_parser
                            and not reasoning_end_arr[i]
                            and not reasoning_parser.is_reasoning_end(
                                previous_token_ids
                            )
                        ):
890
891
                            assert reasoning_parser is not None
                            delta_message = (
892
                                reasoning_parser.extract_reasoning_streaming(
893
894
895
896
897
898
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
899
900
                                )
                            )
901
902
903
904
                            # When encountering think end id in delta_token_ids
                            # or think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
905
                            # Only keep 'content', remove 'reasoning'.
906
                            if reasoning_parser.is_reasoning_end(
907
908
909
910
911
912
913
                                as_list(output.token_ids)
                            ) or (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
914
                                reasoning_end_arr[i] = True
915
916
917
918
919
920
921
922
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
923
                            if self.reasoning_parser:
924
925
926
                                delta_text = previous_text + delta_text
                                current_text = ""

927
928
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
929
930
931
                                    function=DeltaFunctionCall(arguments=delta_text),
                                    index=i,
                                )
932
933
                            else:
                                delta_tool_call = DeltaToolCall(
934
                                    id=make_tool_call_id(),
935
936
937
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
938
939
940
941
                                        arguments=delta_text,
                                    ),
                                    index=i,
                                )
942
943
                                function_name_returned[i] = True

944
945
946
947
948
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    delta_tool_call,
                                ]
                            )
949
                            tools_streamed[i] = True
950

951
952
953
954
955
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]
956
957
958
959
960
961
962
963
964
                        output_token_ids = as_list(output.token_ids)

                        if (
                            self.reasoning_parser is not None
                            and not reasoning_end_arr[i]
                            and res.prompt_token_ids
                            and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
                        ):
                            reasoning_end_arr[i] = True
965

966
967
                        if self.reasoning_parser and not reasoning_end_arr[i]:
                            delta_message = (
968
                                reasoning_parser.extract_reasoning_streaming(
969
970
971
972
973
974
975
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output_token_ids,
                                )
976
                            )
977
978
979
980
981
982
983
984
985
                            if reasoning_parser.is_reasoning_end(output_token_ids):
                                reasoning_end_arr[i] = True
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    # reasoning ended
                                    current_text = ""

986
                        else:
987
                            # either finished reasoning or no reasoning at all
988
                            content = current_text
989
990
991
992
993
994
995
996
997

                            delta_message, function_name_returned[i] = (
                                self.extract_tool_call_required_streaming(
                                    previous_text=previous_text,
                                    current_text=content,
                                    delta_text=delta_text,
                                    function_name_returned=fn_name_returned,
                                    tool_call_idx=history_tool_call_cnt,
                                )
998
                            )
999
1000
1001
1002
1003
1004
1005
                            if (
                                delta_message
                                and delta_message.tool_calls
                                and delta_message.tool_calls[0].id is not None
                            ):
                                history_tool_call_cnt += 1
                                tools_streamed[i] = True
1006

1007
1008
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
1009
                    elif tool_choice_auto and self.reasoning_parser:
1010
1011
1012
1013
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
1014
                        output_token_ids = as_list(output.token_ids)
1015
                        if not reasoning_end_arr[i]:
1016
1017
1018
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
1019
1020
1021
1022
1023
1024
                            if (
                                res.prompt_token_ids
                                and reasoning_parser.is_reasoning_end(
                                    res.prompt_token_ids
                                )
                            ):
1025
                                reasoning_end_arr[i] = True
1026
                                current_token_ids = output_token_ids
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
                                # Don't update current_text, keep it as is from delta
                            else:
                                delta_message = (
                                    reasoning_parser.extract_reasoning_streaming(
                                        previous_text,
                                        current_text,
                                        delta_text,
                                        previous_token_ids,
                                        current_token_ids,
                                        output_token_ids,
1037
1038
                                    )
                                )
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055

                                # When encountering think end id in delta_token_ids,
                                # set reasoning status to end.
                                # Remove the text and token ids related
                                # to 'reasoning'.
                                if reasoning_parser.is_reasoning_end(output_token_ids):
                                    reasoning_end_arr[i] = True
                                    current_token_ids = (
                                        reasoning_parser.extract_content_ids(
                                            output_token_ids
                                        )
                                    )
                                    if delta_message and delta_message.content:
                                        current_text = delta_message.content
                                        delta_message.content = None
                                    else:
                                        current_text = ""
1056
1057

                        # handle tool calls only after reasoning is done,
1058
                        if reasoning_end_arr[i]:
1059
                            delta_token_ids = output_token_ids
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

1070
                            delta_message = tool_parser.extract_tool_calls_streaming(
1071
1072
                                previous_text=previous_text,
                                current_text=current_text,
1073
                                delta_text=delta_text,
1074
1075
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
                                delta_token_ids=delta_token_ids,
                                request=request,
                            )
                            if delta_message and delta_message.tool_calls:
                                tools_streamed[i] = True
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
                        delta_message = tool_parser.extract_tool_calls_streaming(
                            previous_text=previous_text,
                            current_text=current_text,
                            delta_text=delta_text,
                            previous_token_ids=previous_token_ids,
                            current_token_ids=current_token_ids,
                            delta_token_ids=output.token_ids,
                            request=request,
                        )
1093
1094
                        if delta_message and delta_message.tool_calls:
                            tools_streamed[i] = True
1095

1096
                    # when only reasoning
1097
                    elif self.reasoning_parser:
1098
1099
1100
1101
1102
1103
1104
                        delta_message = reasoning_parser.extract_reasoning_streaming(
                            previous_text,
                            current_text,
                            delta_text,
                            previous_token_ids,
                            current_token_ids,
                            output.token_ids,
1105
                        )
1106
                    # handle streaming just a content delta
1107
1108
1109
                    else:
                        delta_message = DeltaMessage(content=delta_text)

1110
                    # update the previous values for the next iteration
1111
1112
1113
                    if (
                        tool_choice_auto or self.reasoning_parser
                    ) and not self.use_harmony:
1114
1115
1116
1117
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
1118
1119
1120
1121
                    else:
                        # Update for comprehensive logging even in simple case
                        assert previous_texts is not None
                        previous_texts[i] += delta_text
1122

1123
                    # set the previous values for the next iteration
1124
                    previous_num_tokens[i] += len(output.token_ids)
1125
1126
1127
1128
1129
1130

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
1131
1132
1133
1134
1135
1136
1137
                        # NOTE: If return_token_ids is enabled, we still need to
                        # send a chunk with token_ids even if delta_message is None
                        # to ensure all tokens are included in the response
                        if (
                            output.finish_reason is None
                            and not request.return_token_ids
                        ):
1138
                            continue
1139
                        delta_message = DeltaMessage()
1140

1141
1142
1143
1144
1145
1146
1147
1148
1149
                    # Log streaming delta if output logging is enabled
                    if self.enable_log_outputs and self.request_logger:
                        delta_content = ""
                        if delta_message.content:
                            delta_content = delta_message.content
                        elif delta_message.tool_calls:
                            delta_content = "".join(
                                tc.function.arguments
                                for tc in delta_message.tool_calls
1150
1151
                                if tc.function and tc.function.arguments
                            )
1152
1153
1154
1155
1156

                        if delta_content:
                            self.request_logger.log_outputs(
                                request_id=request_id,
                                outputs=delta_content,
1157
                                output_token_ids=as_list(output.token_ids),
1158
1159
1160
1161
1162
                                finish_reason=output.finish_reason,
                                is_streaming=True,
                                delta=True,
                            )

1163
1164
1165
1166
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1167
                            delta=delta_message,
1168
                            logprobs=logprobs,
1169
                            finish_reason=None,
1170
1171
1172
1173
1174
1175
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1176
1177

                    # if the model is finished generating
1178
                    else:
1179
1180
1181
1182
                        # check for error finish reason and abort streaming
                        # finish_reason='error' indicates a retryable error
                        self._raise_if_error(output.finish_reason, request_id)

1183
1184
1185
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
1186
                        # only happens if we are NOT using structured outputs
1187
                        auto_tools_called = False
1188
                        if tool_parser:
1189
1190
1191
1192
1193
1194
                            auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
                            index = (
                                len(tool_parser.prev_tool_call_arr) - 1
                                if auto_tools_called
                                else 0
                            )
1195
1196
1197
                        else:
                            index = 0

1198
1199
1200
1201
1202
1203
                        if (
                            self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output
                            )
                            and tool_parser
                        ):
1204
                            latest_delta_len = 0
1205
1206
                            if (
                                isinstance(
1207
                                    delta_message.tool_calls[0].function,
1208
1209
1210
1211
1212
                                    DeltaFunctionCall,
                                )
                            ) and isinstance(
                                delta_message.tool_calls[0].function.arguments, str
                            ):
1213
                                latest_delta_len = len(
1214
1215
                                    delta_message.tool_calls[0].function.arguments
                                )
1216

1217
1218
1219
1220
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
1221
1222
1223
1224
                                    "arguments", {}
                                ),
                                ensure_ascii=False,
                            )
1225

1226
                            # get what we've streamed so far for arguments
1227
                            # for the current tool
1228
1229
                            actual_call = tool_parser.streamed_args_for_tool[index]
                            if latest_delta_len > 0:
1230
                                actual_call = actual_call[:-latest_delta_len]
1231
1232

                            # check to see if there's anything left to stream
1233
                            remaining_call = expected_call.replace(actual_call, "", 1)
1234
                            # set that as a delta message
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
                            delta_message = DeltaMessage(
                                tool_calls=[
                                    DeltaToolCall(
                                        index=index,
                                        function=DeltaFunctionCall(
                                            arguments=remaining_call
                                        ).model_dump(exclude_none=True),
                                    )
                                ]
                            )
1245

1246
                        # Send the finish response for each request.n only once
1247
1248
1249
1250
                        # In OpenAI's API, when a tool is called, the
                        # finish_reason is:
                        # "tool_calls" for "auto" or "required" tool calls,
                        # and "stop" for named tool calls.
1251
1252
                        if (
                            auto_tools_called
1253
                            or (tools_streamed[i] and not tool_choice_function_name)
1254
1255
                            or (self.use_harmony and harmony_tools_streamed[i])
                        ):
1256
1257
                            finish_reason_ = "tool_calls"
                        else:
1258
1259
1260
                            finish_reason_ = (
                                output.finish_reason if output.finish_reason else "stop"
                            )
1261
1262
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
1263
                            delta=delta_message,
1264
                            logprobs=logprobs,
1265
                            finish_reason=finish_reason_,
1266
                            stop_reason=output.stop_reason,
1267
1268
1269
1270
1271
1272
                            token_ids=(
                                as_list(output.token_ids)
                                if request.return_token_ids
                                else None
                            ),
                        )
1273

1274
                        finish_reason_sent[i] = True
1275

1276
                    choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1277
1278
1279
1280
1281
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
1282
1283
                        model=model_name,
                    )
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

1294
                    data = chunk.model_dump_json(exclude_unset=True)
1295
1296
                    yield f"data: {data}\n\n"

1297
1298
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
1299
1300
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
1301
1302
1303
1304
1305
                final_usage = UsageInfo(
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
                )
1306
1307
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
1308
1309
                        cached_tokens=num_cached_tokens
                    )
1310
1311
1312
1313
1314
1315
1316

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
1317
1318
1319
1320
1321
                    usage=final_usage,
                )
                final_usage_data = final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True
                )
1322
                yield f"data: {final_usage_data}\n\n"
1323

1324
1325
1326
1327
1328
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
1329
1330
1331
1332
1333
1334
1335
1336
1337
                total_tokens=num_prompt_tokens + num_completion_tokens,
            )

            # Log complete streaming response if output logging is enabled
            if self.enable_log_outputs and self.request_logger:
                # Log the complete response for each choice
                for i in range(num_choices):
                    full_text = (
                        previous_texts[i]
1338
1339
                        if previous_texts and i < len(previous_texts)
                        else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
1340
1341
1342
1343
                    )
                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=full_text,
1344
                        output_token_ids=None,  # Consider also logging all token IDs
1345
1346
1347
1348
                        finish_reason="streaming_complete",
                        is_streaming=True,
                        delta=False,
                    )
1349

1350
1351
        except GenerationError as e:
            yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
1352
        except Exception as e:
1353
            # TODO: Use a vllm-specific Validation Error
1354
            logger.exception("Error in chat completion stream generator.")
1355
1356
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
1357
1358
1359
1360
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
1361
1362
1363
1364
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
1365
        model_name: str,
1366
        conversation: list[ConversationMessage],
1367
        tokenizer: TokenizerLike | None,
1368
        request_metadata: RequestResponseMetadata,
1369
    ) -> ErrorResponse | ChatCompletionResponse:
1370
        created_time = int(time.time())
1371
        final_res: RequestOutput | None = None
1372

1373
1374
1375
1376
1377
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
1378
1379
1380
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
1381

1382
1383
        assert final_res is not None

1384
        choices: list[ChatCompletionResponseChoice] = []
1385
        if self.tool_call_id_type == "kimi_k2":
1386
1387
1388
            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
        else:
            history_tool_call_cnt = 0
1389

1390
1391
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
1392
1393
1394
            # check for error finish reason and raise GenerationError
            # finish_reason='error' indicates a retryable request-level internal error
            self._raise_if_error(output.finish_reason, request_id)
1395
            token_ids = output.token_ids
1396
            out_logprobs = output.logprobs
1397
            tool_call_info = None
1398

1399
1400
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
1401
                logprobs = self._create_chat_logprobs(
1402
                    token_ids=token_ids,
1403
                    top_logprobs=out_logprobs,
1404
                    num_output_top_logprobs=request.top_logprobs,
1405
                    tokenizer=tokenizer,
1406
                    return_as_token_id=request.return_tokens_as_token_ids,
1407
1408
1409
                )
            else:
                logprobs = None
1410
1411

            if self.use_harmony:
1412
                reasoning, content, _ = parse_chat_output(token_ids)
1413
                if not request.include_reasoning:
1414
                    reasoning = None
1415

1416
                if self.tool_parser is not None:
1417
1418
1419
1420
1421
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1422
1423
1424
1425
1426
1427
1428
                    tool_parser = self.tool_parser(tokenizer)
                    # NOTE: We use token_ids for openai tool parser
                    tool_call_info = tool_parser.extract_tool_calls(
                        "",
                        request=request,
                        token_ids=token_ids,  # type: ignore
                    )
1429
                    content = tool_call_info.content
1430
1431
                    message = ChatMessage(
                        role=role,
1432
                        reasoning=reasoning,
1433
1434
1435
1436
1437
1438
                        content=content,
                        tool_calls=tool_call_info.tool_calls,
                    )
                else:
                    message = ChatMessage(
                        role=role,
1439
                        reasoning=reasoning,
1440
1441
                        content=content,
                    )
1442
1443
1444
1445
1446

                choice_data = ChatCompletionResponseChoice(
                    index=output.index,
                    message=message,
                    logprobs=logprobs,
1447
1448
1449
1450
1451
1452
1453
                    finish_reason=(
                        "tool_calls"
                        if (tool_call_info is not None and tool_call_info.tools_called)
                        else output.finish_reason
                        if output.finish_reason
                        else "stop"
                    ),
1454
                    stop_reason=output.stop_reason,
1455
1456
1457
                    token_ids=(
                        as_list(output.token_ids) if request.return_token_ids else None
                    ),
1458
1459
1460
                )
                choices.append(choice_data)
                continue
1461

1462
            if self.reasoning_parser:
1463
                try:
1464
1465
1466
1467
1468
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1469
1470
1471
1472
                    reasoning_parser = self.reasoning_parser(
                        tokenizer,
                        chat_template_kwargs=request.chat_template_kwargs,  # type: ignore
                    )
1473
1474
1475
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
1476
1477
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
1478
                reasoning, content = reasoning_parser.extract_reasoning(
1479
1480
                    output.text, request=request
                )
1481
                if not request.include_reasoning:
1482
                    reasoning = None
1483
            else:
1484
                reasoning = None
1485
                content = output.text
1486

1487
            auto_tools_called = False
1488
1489
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
            tool_calls, content = self._parse_tool_calls_from_content(
                request=request,
                tokenizer=tokenizer,
                content=content,
                enable_auto_tools=self.enable_auto_tools,
                tool_parser_cls=self.tool_parser,
            )
            tool_call_class = (
                MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
            )
1500
1501
1502
1503
            if (not self.enable_auto_tools or not self.tool_parser) and (
                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
                and request.tool_choice != "required"
            ):
1504
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1505
1506

            # if the request uses tools and specified a tool choice
1507
1508
1509
1510
            elif (
                request.tool_choice
                and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
            ):
1511
                assert tool_calls is not None and len(tool_calls) > 0
1512
1513
                message = ChatMessage(
                    role=role,
1514
                    reasoning=reasoning,
1515
                    content="",
1516
                    tool_calls=[tool_call_class(function=tc) for tc in tool_calls],
1517
                )
1518

1519
            elif request.tool_choice and request.tool_choice == "required":
1520
1521
                tool_call_class_items = []
                assert tool_calls is not None and len(tool_calls) > 0
1522
                for tool_call in tool_calls:
1523
1524
1525
1526
1527
1528
1529
1530
                    tool_call_class_items.append(
                        tool_call_class(
                            id=make_tool_call_id(
                                id_type=self.tool_call_id_type,
                                func_name=tool_call.name,
                                idx=history_tool_call_cnt,
                            ),
                            function=tool_call,
1531
1532
                        )
                    )
1533
                    history_tool_call_cnt += 1
1534
1535
1536
                message = ChatMessage(
                    role=role,
                    content="",
1537
                    tool_calls=tool_call_class_items,
1538
                    reasoning=reasoning,
1539
                )
1540

1541
1542
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1543
            elif not request.tool_choice or request.tool_choice == "none":
1544
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1545
1546

            # handle when there are tools and tool choice is auto
1547
1548
1549
1550
1551
1552
            elif (
                request.tools
                and (request.tool_choice == "auto" or request.tool_choice is None)
                and self.enable_auto_tools
                and self.tool_parser
            ):
1553
1554
1555
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
1556
1557
                auto_tools_called = tool_calls is not None and len(tool_calls) > 0
                if tool_calls:
1558
1559
                    message = ChatMessage(
                        role=role,
1560
                        reasoning=reasoning,
1561
1562
1563
1564
1565
1566
1567
1568
                        content=content,
                        tool_calls=[
                            ToolCall(
                                function=tc,
                                type="function",
                            )
                            for tc in tool_calls
                        ],
1569
                    )
1570
1571
1572
1573

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1574
1575
1576
1577
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
1578
1579
                    if content and len(content) > 0:
                        ret_content = content
1580
1581
                    message = ChatMessage(
                        role=role,
1582
                        reasoning=reasoning,
1583
1584
                        content=ret_content,
                    )
1585
1586
1587
1588
1589
1590

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
1591
1592
                    "completion."
                )
1593
                message = ChatMessage(role=role, reasoning=reasoning, content=content)
1594
1595
1596
1597
1598
1599
1600
1601
            # In OpenAI's API, when a tool is called, the finish_reason is:
            # "tool_calls" for "auto" or "required" tool calls,
            # and "stop" for named tool calls.
            is_finish_reason_tool_calls = auto_tools_called or (
                request.tool_choice
                and request.tool_choice == "required"
                and output.finish_reason == "stop"
            )
1602

1603
1604
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1605
                message=message,
1606
                logprobs=logprobs,
1607
1608
1609
1610
1611
                finish_reason="tool_calls"
                if is_finish_reason_tool_calls
                else output.finish_reason
                if output.finish_reason
                else "stop",
1612
                stop_reason=output.stop_reason,
1613
1614
1615
                token_ids=(
                    as_list(output.token_ids) if request.return_token_ids else None
                ),
1616
            )
1617
            choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
1618

1619
1620
            choices.append(choice_data)

1621
        if request.echo:
1622
            last_msg_content: str | list[dict[str, str]] = ""
1623
1624
1625
1626
1627
            if (
                conversation
                and "content" in conversation[-1]
                and conversation[-1].get("role") == role
            ):
1628
                last_msg_content = conversation[-1]["content"] or ""
1629
            if isinstance(last_msg_content, list):
1630
                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
1631
1632

            for choice in choices:
1633
                full_message = last_msg_content + (choice.message.content or "")
1634
1635
                choice.message.content = full_message

1636
        assert final_res.prompt_token_ids is not None
1637
        num_prompt_tokens = len(final_res.prompt_token_ids)
1638
1639
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1640
        num_generated_tokens = sum(
1641
1642
1643
1644
1645
1646
1647
            len(output.token_ids) for output in final_res.outputs
        )
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
1648
1649
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
1650
1651
                cached_tokens=final_res.num_cached_tokens
            )
1652
1653
1654

        request_metadata.final_usage_info = usage

1655
1656
1657
1658
1659
1660
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1661
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1662
1663
1664
            prompt_token_ids=(
                final_res.prompt_token_ids if request.return_token_ids else None
            ),
Robert Shaw's avatar
Robert Shaw committed
1665
            kv_transfer_params=final_res.kv_transfer_params,
1666
1667
        )

1668
1669
1670
1671
1672
1673
1674
1675
1676
        # Log complete response if output logging is enabled
        if self.enable_log_outputs and self.request_logger:
            for choice in choices:
                output_text = ""
                if choice.message.content:
                    output_text = choice.message.content
                elif choice.message.tool_calls:
                    # For tool calls, log the function name and arguments
                    tool_call_descriptions = []
1677
1678
                    for tc in choice.message.tool_calls:
                        if hasattr(tc.function, "name") and hasattr(
1679
1680
                            tc.function, "arguments"
                        ):
1681
                            tool_call_descriptions.append(
1682
1683
                                f"{tc.function.name}({tc.function.arguments})"
                            )
1684
1685
1686
1687
1688
1689
1690
                    tool_calls_str = ", ".join(tool_call_descriptions)
                    output_text = f"[tool_calls: {tool_calls_str}]"

                if output_text:
                    # Get the corresponding output token IDs
                    output_token_ids = None
                    if choice.index < len(final_res.outputs):
1691
                        output_token_ids = final_res.outputs[choice.index].token_ids
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701

                    self.request_logger.log_outputs(
                        request_id=request_id,
                        outputs=output_text,
                        output_token_ids=output_token_ids,
                        finish_reason=choice.finish_reason,
                        is_streaming=False,
                        delta=False,
                    )

1702
        return response
1703
1704

    def _get_top_logprobs(
1705
1706
        self,
        logprobs: dict[int, Logprob],
1707
        top_logprobs: int | None,
1708
        tokenizer: TokenizerLike | None,
1709
1710
        should_return_as_token_id: bool,
    ) -> list[ChatCompletionLogProb]:
1711
        return [
1712
            ChatCompletionLogProb(
1713
1714
1715
1716
1717
1718
1719
1720
                token=(
                    token := self._get_decoded_token(
                        p[1],
                        p[0],
                        tokenizer,
                        return_as_token_id=should_return_as_token_id,
                    )
                ),
1721
1722
                logprob=max(p[1].logprob, -9999.0),
                bytes=list(token.encode("utf-8", errors="replace")),
1723
1724
            )
            for i, p in enumerate(logprobs.items())
1725
            if (top_logprobs and i < top_logprobs or top_logprobs == -1)
1726
1727
1728
1729
1730
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1731
        top_logprobs: GenericSequence[dict[int, Logprob] | None],
1732
        tokenizer: TokenizerLike | None,
1733
1734
        num_output_top_logprobs: int | None = None,
        return_as_token_id: bool | None = None,
1735
1736
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1737
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1738

1739
1740
1741
1742
1743
        should_return_as_token_id = (
            return_as_token_id
            if return_as_token_id is not None
            else self.return_tokens_as_token_ids
        )
1744
1745
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1746
            if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
1747
                if should_return_as_token_id:
1748
                    token = f"token_id:{token_id}"
1749
                else:
1750
1751
1752
1753
1754
                    if tokenizer is None:
                        raise ValueError(
                            "Tokenizer not available when `skip_tokenizer_init=True`"
                        )

1755
                    token = tokenizer.decode(token_id)
1756

1757
1758
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1759
                        token=token,
1760
                        bytes=list(token.encode("utf-8", errors="replace")),
1761
1762
                    )
                )
1763
            else:
1764
1765
1766
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1767
1768
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1769
                        token=self._get_decoded_token(
1770
1771
1772
                            step_token,
                            token_id,
                            tokenizer,
1773
                            should_return_as_token_id,
1774
1775
                        ),
                        logprob=max(step_token.logprob, -9999.0),
1776
1777
1778
1779
1780
                        bytes=(
                            None
                            if step_decoded is None
                            else list(step_decoded.encode("utf-8", errors="replace"))
                        ),
1781
                        top_logprobs=self._get_top_logprobs(
1782
1783
1784
1785
1786
1787
1788
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                            should_return_as_token_id,
                        ),
                    )
                )
1789
1790

        return ChatCompletionLogProbs(content=logprobs_content)
1791

1792
    def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
1793
1794
1795
1796
1797
1798
1799
1800
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
1801
1802
1803
1804
1805
1806
        return (
            request.tools
            and self.tool_parser
            and self.enable_auto_tools
            and request.tool_choice in ["auto", None]
        )
1807
1808
1809

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
1810
        delta_message: DeltaMessage | None,
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1822
            output.finish_reason is not None
1823
1824
1825
1826
1827
            and self.enable_auto_tools
            and self.tool_parser
            and delta_message
            and delta_message.tool_calls
            and delta_message.tool_calls[0]
1828
1829
1830
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )
1831
1832
1833
1834
1835
1836
1837

    def _make_request_with_harmony(
        self,
        request: ChatCompletionRequest,
    ):
        messages: list[OpenAIMessage] = []

1838
1839
1840
1841
1842
        # because of issues with pydantic we need to potentially
        # re-serialize the tool_calls field of the request
        # for more info: see comment in `maybe_serialize_tool_calls`
        maybe_serialize_tool_calls(request)

1843
1844
1845
1846
1847
1848
1849
1850
        # Add system message.
        # NOTE: In Chat Completion API, browsing is enabled by default
        # if the model supports it. TODO: Support browsing.
        assert not self.supports_browsing
        assert not self.supports_code_interpreter
        sys_msg = get_system_message(
            reasoning_effort=request.reasoning_effort,
            browser_description=None,
1851
            python_description=None,
1852
1853
            with_custom_tools=request.tools is not None,
        )
1854
1855
1856
        messages.append(sys_msg)

        # Add developer message.
1857
        dev_msg = get_developer_message(tools=request.tools)
1858
1859
1860
        messages.append(dev_msg)

        # Add user message.
1861
        messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1862
1863
1864

        # Render prompt token ids.
        prompt_token_ids = render_for_completion(messages)
1865
        engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
1866
1867
1868
1869
1870

        # Add cache_salt if provided in the request
        if request.cache_salt is not None:
            engine_prompt["cache_salt"] = request.cache_salt

1871
        return messages, [engine_prompt]