serving_chat.py 55.6 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import asyncio
4
import json
5
import re
6
import time
7
8
9
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Callable, Final, Optional, Union
10

11
import jinja2
12
import partial_json_parser
13
from fastapi import Request
14
from pydantic import TypeAdapter
15

16
from vllm.config import ModelConfig
17
from vllm.engine.protocol import EngineClient
18
19
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
                                         ConversationMessage)
20
from vllm.entrypoints.logger import RequestLogger
21
from vllm.entrypoints.openai.protocol import (
22
23
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
24
    ChatCompletionRequest, ChatCompletionResponse,
25
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
26
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
27
28
    DeltaToolCall, ErrorResponse, FunctionCall, FunctionDefinition,
    PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo)
29
30
from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
                                                    clamp_prompt_logprobs)
31
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
32
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
33
34
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
    MistralToolCall)
35
from vllm.logger import init_logger
36
from vllm.outputs import CompletionOutput, RequestOutput
37
from vllm.reasoning import ReasoningParser, ReasoningParserManager
38
from vllm.sampling_params import BeamSearchParams, SamplingParams
39
from vllm.sequence import Logprob
40
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
41
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
42
43
                                                truncate_tool_call_ids,
                                                validate_request_params)
44
45
46
47
48
49

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

50
51
52
53
    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
54
        models: OpenAIServingModels,
55
56
57
58
59
60
        response_role: str,
        *,
        request_logger: Optional[RequestLogger],
        chat_template: Optional[str],
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
61
62
        enable_reasoning: bool = False,
        reasoning_parser: Optional[str] = None,
63
64
65
66
        enable_auto_tools: bool = False,
        tool_parser: Optional[str] = None,
        enable_prompt_tokens_details: bool = False,
    ) -> None:
67
        super().__init__(engine_client=engine_client,
68
                         model_config=model_config,
69
                         models=models,
70
71
                         request_logger=request_logger,
                         return_tokens_as_token_ids=return_tokens_as_token_ids)
72

73
        self.response_role = response_role
74
75
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
76

77
78
79
80
81
82
83
84
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

85
86
87
88
89
90
91
92
93
94
95
96
        self.enable_reasoning: bool = enable_reasoning
        self.reasoning_parser: Optional[Callable[[AnyTokenizer],
                                                 ReasoningParser]] = None
        if self.enable_reasoning:
            try:
                self.reasoning_parser = (
                    ReasoningParserManager.get_reasoning_parser(
                        reasoning_parser))
            except Exception as e:
                raise TypeError("Error: --enable-reasoning requires "
                                f"reasoning_parser:'{reasoning_parser}' "
                                "which has not been registered") from e
97
98
        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
99
            try:
100
101
102
103
104
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
105
106
107
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
108
                raise TypeError("Error: --enable-auto-tool-choice requires "
109
110
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
111

112
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
113
114
115
        self.default_sampling_params = (
            self.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
116
117
118
119
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
            logger.info("Using default chat sampling params from %s: %s",
                        source, self.default_sampling_params)
120

121
    async def create_chat_completion(
122
123
        self,
        request: ChatCompletionRequest,
124
125
126
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
127
128
        """
        Chat Completion API similar to OpenAI's API.
129

130
131
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
132
        Chat Completion API.
133
134
135
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
136
            logger.error("Error with model %s", error_check_ret)
137
138
            return error_check_ret

139
140
141
142
143
144
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

145
        try:
146
147
148
149
150
            (
                lora_request,
                prompt_adapter_request,
            ) = self._maybe_get_adapters(request)

151
            model_name = self._get_model_name(request.model, lora_request)
152

153
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
154

155
156
            tool_parser = self.tool_parser

157
            if isinstance(tokenizer, MistralTokenizer):
158
159
160
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
161
                maybe_serialize_tool_calls(request)
162
                truncate_tool_call_ids(request)
163
                validate_request_params(request)
164

165
166
167
168
169
170
171
172
173
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
                    and not isinstance(tokenizer, MistralTokenizer)):
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
174

175
176
177
178
            tool_dicts = None if request.tools is None else [
                tool.model_dump() for tool in request.tools
            ]

179
180
181
182
183
184
185
186
187
            (
                conversation,
                request_prompts,
                engine_prompts,
            ) = await self._preprocess_chat(
                request,
                tokenizer,
                request.messages,
                chat_template=request.chat_template or self.chat_template,
188
                chat_template_content_format=self.chat_template_content_format,
189
190
191
192
193
194
195
196
197
                add_generation_prompt=request.add_generation_prompt,
                continue_final_message=request.continue_final_message,
                tool_dicts=tool_dicts,
                documents=request.documents,
                chat_template_kwargs=request.chat_template_kwargs,
                tool_parser=tool_parser,
                truncate_prompt_tokens=request.truncate_prompt_tokens,
                add_special_tokens=request.add_special_tokens,
            )
198
199
        except (ValueError, TypeError, RuntimeError,
                jinja2.TemplateError) as e:
200
201
            logger.exception("Error in preprocessing prompt inputs")
            return self.create_error_response(str(e))
202

203
204
        request_id = "chatcmpl-" \
                     f"{self._base_request_id(raw_request, request.request_id)}"
205
206
207
208
209

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

210
        # Schedule the request and get the result generator.
211
        generators: list[AsyncGenerator[RequestOutput, None]] = []
212
        try:
213
214
215
216
217
218
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
                default_max_tokens = self.max_model_len - len(
                    engine_prompt["prompt_token_ids"])
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
219
                        default_max_tokens, self.default_sampling_params)
220
221
                else:
                    sampling_params = request.to_sampling_params(
222
                        default_max_tokens,
223
                        self.model_config.logits_processor_pattern,
224
                        self.default_sampling_params)
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
                                 lora_request=lora_request,
                                 prompt_adapter_request=prompt_adapter_request)

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        prompt_adapter_request=prompt_adapter_request,
                        priority=request.priority,
                    )

                generators.append(generator)
253
        except ValueError as e:
254
            # TODO: Use a vllm-specific Validation Error
255
256
            return self.create_error_response(str(e))

257
258
259
        assert len(generators) == 1
        result_generator, = generators

260
261
262
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
263
264
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
265

266
267
        try:
            return await self.chat_completion_full_generator(
268
269
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
270
271
272
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
273
274
275
276

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
277
        return request.messages[-1]["role"]
278

279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
    @staticmethod
    def _bracket_level(s: str, opening='{', closing='}') -> int:
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
    def _filter_delta_text(delta_text: str,
                           previous_text: str) -> tuple[str, bool]:
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
            if c == '{':
                bracket_level += 1
                passed_zero = bracket_level == 0
            elif c == '}':
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
                if c == ',':
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
        current_text: str,
        delta_text: str,
        function_name_returned: bool,
    ) -> tuple[Optional[DeltaMessage], bool]:
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
            logger.debug('not enough tokens to parse into JSON yet')
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
                delta_text, previous_text)
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
            if not finishes_previous_tool and ("name" not in current_tool_call
                                               or "parameters"
                                               not in current_tool_call):
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
                    param_match = re.search(r'.*"parameters":\s*(.*)',
                                            current_text)
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
                        arguments, previous_text)

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
                    if (finishes_previous_tool
                            and "parameters" not in current_tool_call):
                        current_tool_call = obj[-2]

                    function_name_returned = True
                    delta_message = DeltaMessage(tool_calls=[
                        DeltaToolCall(function=DeltaFunctionCall(
                            name=current_tool_call["name"],
                            arguments=arguments),
                                      index=len(obj) - 1,
                                      type="function")
                    ])

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
                        delta_text, previous_text)

                    if delta_text != "":
                        delta_message = DeltaMessage(tool_calls=[
                            DeltaToolCall(
                                function=DeltaFunctionCall(
                                    # OpenAI API returns None
                                    # instead of name every time
                                    name=None,
                                    arguments=delta_text),
                                index=len(obj) - 1,
                                type="function")
                        ])
                    else:
                        delta_message = None

        return delta_message, function_name_returned

395
    async def chat_completion_stream_generator(
396
397
398
399
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
400
        model_name: str,
401
        conversation: list[ConversationMessage],
402
        tokenizer: AnyTokenizer,
403
        request_metadata: RequestResponseMetadata,
404
    ) -> AsyncGenerator[str, None]:
405
        created_time = int(time.time())
406
        chunk_object_type: Final = "chat.completion.chunk"
407
        first_iteration = True
408
409

        # Send response for each token for each request.n (index)
410
411
412
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
413
        num_prompt_tokens = 0
414
        num_cached_tokens = None
415
416
417
418
419
420
421
422
423
424
425

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

426
427
428
        should_stream_with_reasoning_parsing = (
            self._should_stream_with_reasoning_parsing(request))

429
        all_previous_token_ids: Optional[list[list[int]]]
430
        function_name_returned: Optional[list[bool]] = None
431
432
433
434

        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
        if tool_choice_auto or should_stream_with_reasoning_parsing:
435
436
437
            # These are only required in "auto" tool choice case
            previous_texts = [""] * num_choices
            all_previous_token_ids = [[]] * num_choices
438
439
440
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
441
442
443
444
        elif request.tool_choice == "required":
            previous_texts = [""] * num_choices
            function_name_returned = [False] * num_choices
            all_previous_token_ids = None
445
446
447
        else:
            previous_texts, all_previous_token_ids = None, None

448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
        try:
            # There is no need to check if the reasoning_parser is None
            # because the should_stream_with_reasoning_parsing check
            # already ensures that the reasoning_parser is not None.
            # but the pre-commit hook requires it.
            if should_stream_with_reasoning_parsing and \
                self.reasoning_parser is not None:
                reasoning_parser = self.reasoning_parser(tokenizer)
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

463
464
465
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
466
                tool_parsers: list[Optional[ToolParser]] = [
467
468
469
470
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
471
        except Exception as e:
472
            logger.exception("Error in tool parser creation.")
473
474
475
476
477
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

478
479
480
481
482
483
484
485
        stream_options = request.stream_options
        if stream_options:
            include_usage = stream_options.include_usage
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

486
487
        try:
            async for res in result_generator:
488
489
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
490
491
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
492

493
494
495
496
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
497
                    num_cached_tokens = res.num_cached_tokens
498
499
                    # Send first response for each request.n (index) with
                    # the role
500
                    role = self.get_chat_request_role(request)
501
502
503

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
504
                    for i in range(num_choices):
505
506
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
507
508
509
510
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
511
512
513
514
515
516
517
518
                            logprobs=None,
                            finish_reason=None)
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
                            model=model_name)
519

520
521
522
523
524
525
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
526

527
528
529
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

530
531
                    # Send response to echo the input portion of the
                    # last message
532
                    if request.echo:
533
                        last_msg_content: Union[str, list[dict[str, str]]] = ""
534
535
536
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
537
538

                        if last_msg_content:
539
                            for i in range(num_choices):
540
541
542
543
544
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
545
                                        logprobs=None,
546
                                        finish_reason=None))
547
548
549
550
551
552
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
553
554
555
556
557
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
558

559
560
561
562
563
564
565
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
566
                    tool_parser = tool_parsers[i]
567
568
569
570

                    if finish_reason_sent[i]:
                        continue

571
                    if request.logprobs and request.top_logprobs is not None:
572
                        assert output.logprobs is not None, (
573
                            "Did not output logprobs")
574
                        logprobs = self._create_chat_logprobs(
575
576
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
577
                            tokenizer=tokenizer,
578
                            num_output_top_logprobs=request.top_logprobs,
579
580
                            return_as_token_id=request.
                            return_tokens_as_token_ids,
581
582
583
584
                        )
                    else:
                        logprobs = None

585
                    delta_text = output.text
586
587
588
589
590
591

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

592
                    delta_message: Optional[DeltaMessage]
593

594
595
                    # just update previous_texts and previous_token_ids
                    if tool_choice_auto or should_stream_with_reasoning_parsing:
596
597
598
599
600
601
602
603
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
                        current_token_ids = previous_token_ids + list(
                            output.token_ids)

604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
                    # handle streaming deltas for tools with named tool_choice
                    if tool_choice_function_name:
                        if (self.enable_reasoning
                                and not reasoning_parser.is_reasoning_end(
                                    previous_token_ids)):
                            assert reasoning_parser is not None
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))
                            # When encountering think end id in delta_token_ids,
                            # process the `content`. Only keep 'content',
                            # remove 'reasoning_content'
                            if reasoning_parser.is_reasoning_end(
                                    list(output.token_ids)):
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
                            if self.enable_reasoning:
                                delta_text = previous_text + delta_text
                                current_text = ""

                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(function=DeltaFunctionCall(
                                    name=tool_choice_function_name,
                                    arguments=delta_text),
                                              index=i)
                            ])

644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        assert function_name_returned is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]

                        delta_message, function_name_returned[i] = (
                            self.extract_tool_call_required_streaming(
                                previous_text=previous_text,
                                current_text=current_text,
                                delta_text=delta_text,
                                function_name_returned=fn_name_returned))

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text

661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
                    elif tool_choice_auto and self.enable_reasoning:
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
                        if not reasoning_end_arr[i]:
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))

                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if reasoning_parser.is_reasoning_end(
                                    list(output.token_ids)):
                                reasoning_end_arr[i] = True
                                current_token_ids =  \
                                    reasoning_parser.extract_content_ids(
                                        list(output.token_ids))
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
                            delta_token_ids = list(output.token_ids)
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

                            delta_message = (
                                tool_parser.extract_tool_calls_streaming(
                                    previous_text=previous_text,
                                    current_text=current_text,
                                    delta_text=delta_text,
                                    previous_token_ids=previous_token_ids,
                                    current_token_ids=current_token_ids,
                                    delta_token_ids=delta_token_ids,
                                    request=request))
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
721
722
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
723
724
                                previous_text=previous_text,
                                current_text=current_text,
725
                                delta_text=delta_text,
726
727
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
728
729
                                delta_token_ids=output.token_ids,
                                request=request))
730
                    # when only reasoning
731
732
733
734
735
736
737
738
739
740
741
                    elif self.enable_reasoning:
                        assert reasoning_parser is not None
                        delta_message = (reasoning_parser.
                                         extract_reasoning_content_streaming(
                                             previous_text,
                                             current_text,
                                             delta_text,
                                             previous_token_ids,
                                             current_token_ids,
                                             output.token_ids,
                                         ))
742
                    # handle streaming just a content delta
743
744
745
                    else:
                        delta_message = DeltaMessage(content=delta_text)

746
747
748
749
750
751
752
                    # update the previous values for the next iteration
                    if tool_choice_auto or should_stream_with_reasoning_parsing:
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids

753
                    # set the previous values for the next iteration
754
                    previous_num_tokens[i] += len(output.token_ids)
755
756
757
758
759
760
761
762

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

763
764
765
766
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
767
                            delta=delta_message,
768
769
                            logprobs=logprobs,
                            finish_reason=None)
770
771

                    # if the model is finished generating
772
                    else:
773
774
775
776
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
777
                        auto_tools_called = False
778
                        if tool_parser:
779
780
781
782
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
783
784
785
786
787
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
788
789
790
791
792
793
794
795
796
797
                            latest_delta_len = 0
                            if ((isinstance(
                                    delta_message.tool_calls[0].function,
                                    DeltaFunctionCall)) and isinstance(
                                        delta_message.tool_calls[0].function.
                                        arguments, str)):
                                latest_delta_len = len(
                                    delta_message.tool_calls[0].function.
                                    arguments)

798
799
800
801
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
802
803
                                    "arguments", {}),
                                ensure_ascii=False)
804

805
                            # get what we've streamed so far for arguments
806
807
808
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]
809
810
                            if (latest_delta_len > 0):
                                actual_call = actual_call[:-latest_delta_len]
811
812
813
814
815
816
817
818
819
820
821
822

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)
                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

823
824
825
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
826
                            delta=delta_message,
827
                            logprobs=logprobs,
828
                            finish_reason=output.finish_reason
829
                            if not auto_tools_called else "tool_calls",
830
                            stop_reason=output.stop_reason)
831

832
                        finish_reason_sent[i] = True
833

834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

                    data = chunk.model_dump_json(exclude_unset=True)
                    yield f"data: {data}\n\n"

853
854
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
855
856
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
857
858
859
860
861
862
863
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
864
865
866
867
868
869
870
871
872
873
874

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
875

876
877
878
879
880
881
882
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
                total_tokens=num_prompt_tokens + num_completion_tokens)

883
        except Exception as e:
884
            # TODO: Use a vllm-specific Validation Error
885
            logger.exception("Error in chat completion stream generator.")
886
887
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
888
889
890
891
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
892
893
894
895
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
896
        model_name: str,
897
        conversation: list[ConversationMessage],
898
        tokenizer: AnyTokenizer,
899
        request_metadata: RequestResponseMetadata,
900
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
901

902
        created_time = int(time.time())
903
        final_res: Optional[RequestOutput] = None
904

905
906
907
908
909
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
910
911
912
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
913

914
915
        assert final_res is not None

916
        choices: list[ChatCompletionResponseChoice] = []
917

918
919
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
920
            token_ids = output.token_ids
921
            out_logprobs = output.logprobs
922

923
924
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
925
                logprobs = self._create_chat_logprobs(
926
                    token_ids=token_ids,
927
                    top_logprobs=out_logprobs,
928
                    num_output_top_logprobs=request.top_logprobs,
929
                    tokenizer=tokenizer,
930
                    return_as_token_id=request.return_tokens_as_token_ids,
931
932
933
934
                )
            else:
                logprobs = None

935
936
937
            should_stream_with_reasoning_parsing = (
                self._should_stream_with_reasoning_parsing(request))

938
939
940
941
            # In the OpenAI API the finish_reason is "tools_called"
            # if the tool choice is auto and the model produced a tool
            # call. The same is not true for named function calls
            auto_tools_called = False
942

943
944
945
946
947
948
949
            if should_stream_with_reasoning_parsing and \
                self.reasoning_parser is not None:
                try:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
950
951
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
952
953
954
                reasoning_content, content = (
                    reasoning_parser.extract_reasoning_content(
                        output.text, request=request))
955
956
957
            else:
                reasoning_content = None
                content = output.text
958

959
960
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
961
962
963
964
            if (not self.enable_auto_tools or not self.tool_parser) and \
                (not isinstance(request.tool_choice,
                                ChatCompletionNamedToolChoiceParam
                                ) and request.tool_choice != "required"):
965
966
967
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
968
969
970

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
971
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
972

973
974
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall
975
976
                message = ChatMessage(
                    role=role,
977
                    reasoning_content=reasoning_content,
978
979
                    content="",
                    tool_calls=[
980
                        tool_call_class(function=FunctionCall(
981
                            name=request.tool_choice.function.name,
982
                            arguments=content))
983
                    ])
984

985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
            elif request.tool_choice and request.tool_choice == "required":
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall

                # the fields of FunctionDefinition are a superset of the
                # tool call outputs and can be used for parsing
                tool_calls = TypeAdapter(
                    list[FunctionDefinition]).validate_json(output.text)
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
                        tool_call_class(function=FunctionCall(
                            name=tool_call.name,
                            arguments=json.dumps(tool_call.parameters)))
                        for tool_call in tool_calls
                    ])

1003
1004
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1005
            elif not request.tool_choice or request.tool_choice == "none":
1006

1007
1008
1009
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1010
1011
1012
1013
1014
1015
1016

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

1017
1018
1019
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
1020
                    logger.exception("Error in tool parser creation.")
1021
1022
                    return self.create_error_response(str(e))

1023
                tool_call_info = tool_parser.extract_tool_calls(
1024
                    content if content is not None else "", request=request)
1025
1026
1027
1028
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
1029
1030
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
1031
                                          reasoning_content=reasoning_content,
1032
1033
1034
1035
1036
1037
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1038
1039
1040
                    message = ChatMessage(role=role,
                                          reasoning_content=reasoning_content,
                                          content=content)
1041
1042
1043
1044
1045
1046
1047

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
1048
1049
1050
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1051

1052
1053
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1054
                message=message,
1055
                logprobs=logprobs,
1056
                finish_reason="tool_calls" if auto_tools_called else
1057
                output.finish_reason if output.finish_reason else "stop",
1058
                stop_reason=output.stop_reason)
1059
1060
            choices.append(choice_data)

1061
        if request.echo:
1062
            last_msg_content: Union[str, list[dict[str, str]]] = ""
1063
1064
            if conversation and "content" in conversation[-1] and conversation[
                    -1].get("role") == role:
1065
                last_msg_content = conversation[-1]["content"] or ""
1066
1067
1068
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
1069
1070

            for choice in choices:
1071
1072
                full_message = last_msg_content + (choice.message.content
                                                   or "")
1073
1074
                choice.message.content = full_message

1075
        assert final_res.prompt_token_ids is not None
1076
        num_prompt_tokens = len(final_res.prompt_token_ids)
1077
1078
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1079
1080
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
1081
1082
1083
1084
1085
1086
1087
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
1088
1089
1090

        request_metadata.final_usage_info = usage

1091
1092
1093
1094
1095
1096
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1097
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
1098
1099
        )

1100
        return response
1101
1102

    def _get_top_logprobs(
1103
            self, logprobs: dict[int, Logprob], top_logprobs: Optional[int],
1104
1105
            tokenizer: AnyTokenizer,
            should_return_as_token_id: bool) -> list[ChatCompletionLogProb]:
1106
        return [
1107
1108
1109
1110
            ChatCompletionLogProb(token=(token := self._get_decoded_token(
                p[1],
                p[0],
                tokenizer,
1111
                return_as_token_id=should_return_as_token_id)),
1112
1113
1114
                                  logprob=max(p[1].logprob, -9999.0),
                                  bytes=list(
                                      token.encode("utf-8", errors="replace")))
1115
1116
1117
1118
1119
1120
1121
            for i, p in enumerate(logprobs.items())
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1122
        top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
1123
        tokenizer: AnyTokenizer,
1124
        num_output_top_logprobs: Optional[int] = None,
1125
        return_as_token_id: Optional[bool] = None,
1126
1127
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1128
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1129

1130
1131
        should_return_as_token_id = return_as_token_id if \
            return_as_token_id is not None else self.return_tokens_as_token_ids
1132
1133
1134
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
            if step_top_logprobs is None:
1135
                token = tokenizer.decode(token_id)
1136
                if should_return_as_token_id:
1137
                    token = f"token_id:{token_id}"
1138

1139
1140
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1141
                        token=token,
1142
1143
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
1144
            else:
1145
1146
1147
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1148
1149
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1150
                        token=self._get_decoded_token(
1151
1152
1153
                            step_token,
                            token_id,
                            tokenizer,
1154
                            should_return_as_token_id,
1155
1156
1157
1158
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
1159
                        top_logprobs=self._get_top_logprobs(
1160
1161
                            step_top_logprobs, num_output_top_logprobs,
                            tokenizer, should_return_as_token_id),
1162
                    ))
1163
1164

        return ChatCompletionLogProbs(content=logprobs_content)
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
    def _should_stream_with_reasoning_parsing(self,
                                              request: ChatCompletionRequest):
        """
            Utility function to check if streamed tokens should go through the
            reasoning parser that was configured.
    
            We only want to do this IF reasoning is enabled and a reasoning 
            parser is configured.
            """
        return self.enable_reasoning and self.reasoning_parser is not None

1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1205
1206
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
1207
1208
1209
1210
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )