serving_chat.py 55 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
9
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Callable, Final, Optional, Union
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from pydantic import TypeAdapter
16

17
from vllm.config import ModelConfig
18
from vllm.engine.protocol import EngineClient
19
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
20
21
                                         ConversationMessage,
                                         random_tool_call_id)
22
from vllm.entrypoints.logger import RequestLogger
23
from vllm.entrypoints.openai.protocol import (
24
25
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
26
    ChatCompletionRequest, ChatCompletionResponse,
27
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
28
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
29
30
    DeltaToolCall, ErrorResponse, FunctionCall, FunctionDefinition,
    PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo)
31
32
from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
                                                    clamp_prompt_logprobs)
33
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
34
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
35
36
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
    MistralToolCall)
37
from vllm.logger import init_logger
38
from vllm.outputs import CompletionOutput, RequestOutput
39
from vllm.reasoning import ReasoningParser, ReasoningParserManager
40
from vllm.sampling_params import BeamSearchParams, SamplingParams
41
from vllm.sequence import Logprob
42
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
43
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
44
45
                                                truncate_tool_call_ids,
                                                validate_request_params)
46
47
48
49
50
51

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

52
53
54
55
    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
56
        models: OpenAIServingModels,
57
58
59
60
61
62
        response_role: str,
        *,
        request_logger: Optional[RequestLogger],
        chat_template: Optional[str],
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
63
        reasoning_parser: str = "",
64
65
66
67
        enable_auto_tools: bool = False,
        tool_parser: Optional[str] = None,
        enable_prompt_tokens_details: bool = False,
    ) -> None:
68
        super().__init__(engine_client=engine_client,
69
                         model_config=model_config,
70
                         models=models,
71
72
                         request_logger=request_logger,
                         return_tokens_as_token_ids=return_tokens_as_token_ids)
73

74
        self.response_role = response_role
75
76
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
77

78
79
80
81
82
83
84
85
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

86
87
        self.reasoning_parser: Optional[Callable[[AnyTokenizer],
                                                 ReasoningParser]] = None
88
        if reasoning_parser:
89
90
91
92
            try:
                self.reasoning_parser = (
                    ReasoningParserManager.get_reasoning_parser(
                        reasoning_parser))
93
                assert self.reasoning_parser is not None
94
            except Exception as e:
95
96
                raise TypeError(
                    f"{reasoning_parser=} has not been registered") from e
97
98
        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
99
            try:
100
101
102
103
104
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
105
106
107
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
108
                raise TypeError("Error: --enable-auto-tool-choice requires "
109
110
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
111

112
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
113
114
115
        self.default_sampling_params = (
            self.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
116
117
118
119
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
            logger.info("Using default chat sampling params from %s: %s",
                        source, self.default_sampling_params)
120

121
    async def create_chat_completion(
122
123
        self,
        request: ChatCompletionRequest,
124
125
126
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
127
128
        """
        Chat Completion API similar to OpenAI's API.
129

130
131
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
132
        Chat Completion API.
133
134
135
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
136
            logger.error("Error with model %s", error_check_ret)
137
138
            return error_check_ret

139
140
141
142
143
144
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

145
        try:
146
147
148
149
150
            (
                lora_request,
                prompt_adapter_request,
            ) = self._maybe_get_adapters(request)

151
            model_name = self._get_model_name(request.model, lora_request)
152

153
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
154

155
156
            tool_parser = self.tool_parser

157
            if isinstance(tokenizer, MistralTokenizer):
158
159
160
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
161
                maybe_serialize_tool_calls(request)
162
                truncate_tool_call_ids(request)
163
                validate_request_params(request)
164

165
166
167
168
169
170
171
172
173
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
                    and not isinstance(tokenizer, MistralTokenizer)):
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
174

175
176
177
178
            tool_dicts = None if request.tools is None else [
                tool.model_dump() for tool in request.tools
            ]

179
180
181
182
183
184
185
186
187
            (
                conversation,
                request_prompts,
                engine_prompts,
            ) = await self._preprocess_chat(
                request,
                tokenizer,
                request.messages,
                chat_template=request.chat_template or self.chat_template,
188
                chat_template_content_format=self.chat_template_content_format,
189
190
191
192
193
194
195
196
197
                add_generation_prompt=request.add_generation_prompt,
                continue_final_message=request.continue_final_message,
                tool_dicts=tool_dicts,
                documents=request.documents,
                chat_template_kwargs=request.chat_template_kwargs,
                tool_parser=tool_parser,
                truncate_prompt_tokens=request.truncate_prompt_tokens,
                add_special_tokens=request.add_special_tokens,
            )
198
199
        except (ValueError, TypeError, RuntimeError,
                jinja2.TemplateError) as e:
200
            logger.exception("Error in preprocessing prompt inputs")
201
            return self.create_error_response(f"{e} {e.__cause__}")
202

203
204
        request_id = "chatcmpl-" \
                     f"{self._base_request_id(raw_request, request.request_id)}"
205
206
207
208
209

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

210
        # Schedule the request and get the result generator.
211
        generators: list[AsyncGenerator[RequestOutput, None]] = []
212
        try:
213
214
215
216
217
218
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
                default_max_tokens = self.max_model_len - len(
                    engine_prompt["prompt_token_ids"])
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
219
                        default_max_tokens, self.default_sampling_params)
220
221
                else:
                    sampling_params = request.to_sampling_params(
222
                        default_max_tokens,
223
                        self.model_config.logits_processor_pattern,
224
                        self.default_sampling_params)
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
                                 lora_request=lora_request,
                                 prompt_adapter_request=prompt_adapter_request)

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
240
                        lora_request=lora_request,
241
242
243
244
245
246
247
248
249
250
251
252
253
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        prompt_adapter_request=prompt_adapter_request,
                        priority=request.priority,
                    )

                generators.append(generator)
254
        except ValueError as e:
255
            # TODO: Use a vllm-specific Validation Error
256
257
            return self.create_error_response(str(e))

258
259
260
        assert len(generators) == 1
        result_generator, = generators

261
262
263
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
264
265
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
266

267
268
        try:
            return await self.chat_completion_full_generator(
269
270
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
271
272
273
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
274
275
276
277

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
278
        return request.messages[-1]["role"]
279

280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
    @staticmethod
    def _bracket_level(s: str, opening='{', closing='}') -> int:
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
    def _filter_delta_text(delta_text: str,
                           previous_text: str) -> tuple[str, bool]:
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
            if c == '{':
                bracket_level += 1
                passed_zero = bracket_level == 0
            elif c == '}':
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
                if c == ',':
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
        current_text: str,
        delta_text: str,
        function_name_returned: bool,
    ) -> tuple[Optional[DeltaMessage], bool]:
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
            logger.debug('not enough tokens to parse into JSON yet')
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
                delta_text, previous_text)
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
            if not finishes_previous_tool and ("name" not in current_tool_call
                                               or "parameters"
                                               not in current_tool_call):
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
                    param_match = re.search(r'.*"parameters":\s*(.*)',
                                            current_text)
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
                        arguments, previous_text)

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
                    if (finishes_previous_tool
                            and "parameters" not in current_tool_call):
                        current_tool_call = obj[-2]

                    function_name_returned = True
                    delta_message = DeltaMessage(tool_calls=[
369
370
371
372
                        DeltaToolCall(id=random_tool_call_id(),
                                      function=DeltaFunctionCall(
                                          name=current_tool_call["name"],
                                          arguments=arguments),
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
                                      index=len(obj) - 1,
                                      type="function")
                    ])

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
                        delta_text, previous_text)

                    if delta_text != "":
                        delta_message = DeltaMessage(tool_calls=[
                            DeltaToolCall(
                                function=DeltaFunctionCall(
                                    # OpenAI API returns None
                                    # instead of name every time
                                    name=None,
                                    arguments=delta_text),
389
                                index=len(obj) - 1)
390
391
392
393
394
395
                        ])
                    else:
                        delta_message = None

        return delta_message, function_name_returned

396
    async def chat_completion_stream_generator(
397
398
399
400
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
401
        model_name: str,
402
        conversation: list[ConversationMessage],
403
        tokenizer: AnyTokenizer,
404
        request_metadata: RequestResponseMetadata,
405
    ) -> AsyncGenerator[str, None]:
406
        created_time = int(time.time())
407
        chunk_object_type: Final = "chat.completion.chunk"
408
        first_iteration = True
409
410

        # Send response for each token for each request.n (index)
411
412
413
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
414
        num_prompt_tokens = 0
415
        num_cached_tokens = None
416
417
418
419
420
421
422
423
424
425
426

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

427
        all_previous_token_ids: Optional[list[list[int]]]
428
        function_name_returned = [False] * num_choices
429
430
431

        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
432
        if tool_choice_auto or self.reasoning_parser:
433
434
435
            # These are only required in "auto" tool choice case
            previous_texts = [""] * num_choices
            all_previous_token_ids = [[]] * num_choices
436
437
438
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
439
440
441
        elif request.tool_choice == "required":
            previous_texts = [""] * num_choices
            all_previous_token_ids = None
442
443
444
        else:
            previous_texts, all_previous_token_ids = None, None

445
        try:
446
            if self.reasoning_parser:
447
448
449
450
451
452
453
                reasoning_parser = self.reasoning_parser(tokenizer)
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
454
455
456
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
457
                tool_parsers: list[Optional[ToolParser]] = [
458
459
460
461
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
462
        except Exception as e:
463
            logger.exception("Error in tool parser creation.")
464
465
466
467
468
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

469
470
471
472
473
474
475
476
        stream_options = request.stream_options
        if stream_options:
            include_usage = stream_options.include_usage
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

477
478
        try:
            async for res in result_generator:
479
480
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
481
482
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
483

484
485
486
487
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
488
                    num_cached_tokens = res.num_cached_tokens
489
490
                    # Send first response for each request.n (index) with
                    # the role
491
                    role = self.get_chat_request_role(request)
492
493
494

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
495
                    for i in range(num_choices):
496
497
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
498
499
500
501
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
502
503
504
505
506
507
508
509
                            logprobs=None,
                            finish_reason=None)
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
                            model=model_name)
510

511
512
513
514
515
516
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
517

518
519
520
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

521
522
                    # Send response to echo the input portion of the
                    # last message
523
                    if request.echo:
524
                        last_msg_content: Union[str, list[dict[str, str]]] = ""
525
526
527
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
528
529

                        if last_msg_content:
530
                            for i in range(num_choices):
531
532
533
534
535
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
536
                                        logprobs=None,
537
                                        finish_reason=None))
538
539
540
541
542
543
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
544
545
546
547
548
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
549

550
551
552
553
554
555
556
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
557
                    tool_parser = tool_parsers[i]
558
559
560
561

                    if finish_reason_sent[i]:
                        continue

562
                    if request.logprobs and request.top_logprobs is not None:
563
                        assert output.logprobs is not None, (
564
                            "Did not output logprobs")
565
                        logprobs = self._create_chat_logprobs(
566
567
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
568
                            tokenizer=tokenizer,
569
                            num_output_top_logprobs=request.top_logprobs,
570
571
                            return_as_token_id=request.
                            return_tokens_as_token_ids,
572
573
574
575
                        )
                    else:
                        logprobs = None

576
                    delta_text = output.text
577
578
579
580
581
582

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

583
                    delta_message: Optional[DeltaMessage]
584

585
                    # just update previous_texts and previous_token_ids
586
                    if tool_choice_auto or self.reasoning_parser:
587
588
589
590
591
592
593
594
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
                        current_token_ids = previous_token_ids + list(
                            output.token_ids)

595
596
                    # handle streaming deltas for tools with named tool_choice
                    if tool_choice_function_name:
597
                        if (self.reasoning_parser
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
                                and not reasoning_parser.is_reasoning_end(
                                    previous_token_ids)):
                            assert reasoning_parser is not None
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))
                            # When encountering think end id in delta_token_ids,
                            # process the `content`. Only keep 'content',
                            # remove 'reasoning_content'
                            if reasoning_parser.is_reasoning_end(
                                    list(output.token_ids)):
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
624
                            if self.reasoning_parser:
625
626
627
                                delta_text = previous_text + delta_text
                                current_text = ""

628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        arguments=delta_text),
                                    index=i)
                            else:
                                delta_tool_call = DeltaToolCall(
                                    id=random_tool_call_id(),
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
                                        arguments=delta_text),
                                    index=i)
                                function_name_returned[i] = True

643
                            delta_message = DeltaMessage(tool_calls=[
644
                                delta_tool_call,
645
646
                            ])

647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]

                        delta_message, function_name_returned[i] = (
                            self.extract_tool_call_required_streaming(
                                previous_text=previous_text,
                                current_text=current_text,
                                delta_text=delta_text,
                                function_name_returned=fn_name_returned))

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text

663
664
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
665
                    elif tool_choice_auto and self.reasoning_parser:
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
                        if not reasoning_end_arr[i]:
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))

                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if reasoning_parser.is_reasoning_end(
                                    list(output.token_ids)):
                                reasoning_end_arr[i] = True
                                current_token_ids =  \
                                    reasoning_parser.extract_content_ids(
                                        list(output.token_ids))
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
                            delta_token_ids = list(output.token_ids)
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

                            delta_message = (
                                tool_parser.extract_tool_calls_streaming(
                                    previous_text=previous_text,
                                    current_text=current_text,
                                    delta_text=delta_text,
                                    previous_token_ids=previous_token_ids,
                                    current_token_ids=current_token_ids,
                                    delta_token_ids=delta_token_ids,
                                    request=request))
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
723
724
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
725
726
                                previous_text=previous_text,
                                current_text=current_text,
727
                                delta_text=delta_text,
728
729
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
730
731
                                delta_token_ids=output.token_ids,
                                request=request))
732
                    # when only reasoning
733
                    elif self.reasoning_parser:
734
735
736
737
738
739
740
741
742
                        delta_message = (reasoning_parser.
                                         extract_reasoning_content_streaming(
                                             previous_text,
                                             current_text,
                                             delta_text,
                                             previous_token_ids,
                                             current_token_ids,
                                             output.token_ids,
                                         ))
743
                    # handle streaming just a content delta
744
745
746
                    else:
                        delta_message = DeltaMessage(content=delta_text)

747
                    # update the previous values for the next iteration
748
                    if tool_choice_auto or self.reasoning_parser:
749
750
751
752
753
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids

754
                    # set the previous values for the next iteration
755
                    previous_num_tokens[i] += len(output.token_ids)
756
757
758
759
760
761
762
763

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

764
765
766
767
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
768
                            delta=delta_message,
769
770
                            logprobs=logprobs,
                            finish_reason=None)
771
772

                    # if the model is finished generating
773
                    else:
774
775
776
777
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
778
                        auto_tools_called = False
779
                        if tool_parser:
780
781
782
783
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
784
785
786
787
788
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
789
790
791
792
793
794
795
796
797
798
                            latest_delta_len = 0
                            if ((isinstance(
                                    delta_message.tool_calls[0].function,
                                    DeltaFunctionCall)) and isinstance(
                                        delta_message.tool_calls[0].function.
                                        arguments, str)):
                                latest_delta_len = len(
                                    delta_message.tool_calls[0].function.
                                    arguments)

799
800
801
802
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
803
804
                                    "arguments", {}),
                                ensure_ascii=False)
805

806
                            # get what we've streamed so far for arguments
807
808
809
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]
810
811
                            if (latest_delta_len > 0):
                                actual_call = actual_call[:-latest_delta_len]
812
813
814
815
816
817
818
819
820
821
822
823

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)
                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

824
825
826
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
827
                            delta=delta_message,
828
                            logprobs=logprobs,
829
                            finish_reason=output.finish_reason
830
                            if not auto_tools_called else "tool_calls",
831
                            stop_reason=output.stop_reason)
832

833
                        finish_reason_sent[i] = True
834

835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

851
                    data = chunk.model_dump_json(exclude_none=True)
852
853
                    yield f"data: {data}\n\n"

854
855
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
856
857
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
858
859
860
861
862
863
864
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
865
866
867
868
869
870
871
872
873
874
875

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
876

877
878
879
880
881
882
883
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
                total_tokens=num_prompt_tokens + num_completion_tokens)

884
        except Exception as e:
885
            # TODO: Use a vllm-specific Validation Error
886
            logger.exception("Error in chat completion stream generator.")
887
888
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
889
890
891
892
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
893
894
895
896
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
897
        model_name: str,
898
        conversation: list[ConversationMessage],
899
        tokenizer: AnyTokenizer,
900
        request_metadata: RequestResponseMetadata,
901
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
902

903
        created_time = int(time.time())
904
        final_res: Optional[RequestOutput] = None
905

906
907
908
909
910
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
911
912
913
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
914

915
916
        assert final_res is not None

917
        choices: list[ChatCompletionResponseChoice] = []
918

919
920
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
921
            token_ids = output.token_ids
922
            out_logprobs = output.logprobs
923

924
925
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
926
                logprobs = self._create_chat_logprobs(
927
                    token_ids=token_ids,
928
                    top_logprobs=out_logprobs,
929
                    num_output_top_logprobs=request.top_logprobs,
930
                    tokenizer=tokenizer,
931
                    return_as_token_id=request.return_tokens_as_token_ids,
932
933
934
                )
            else:
                logprobs = None
935
            auto_tools_called = False
936

937
            if self.reasoning_parser:
938
939
940
941
942
                try:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
943
944
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
945
946
947
                reasoning_content, content = (
                    reasoning_parser.extract_reasoning_content(
                        output.text, request=request))
948
949
950
            else:
                reasoning_content = None
                content = output.text
951

952
953
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
954
955
956
957
            if (not self.enable_auto_tools or not self.tool_parser) and \
                (not isinstance(request.tool_choice,
                                ChatCompletionNamedToolChoiceParam
                                ) and request.tool_choice != "required"):
958
959
960
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
961
962
963

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
964
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
965

966
967
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall
968
969
                message = ChatMessage(
                    role=role,
970
                    reasoning_content=reasoning_content,
971
972
                    content="",
                    tool_calls=[
973
                        tool_call_class(function=FunctionCall(
974
                            name=request.tool_choice.function.name,
975
                            arguments=content))
976
                    ])
977

978
979
980
981
982
983
984
985
986
987
988
989
990
991
            elif request.tool_choice and request.tool_choice == "required":
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall

                # the fields of FunctionDefinition are a superset of the
                # tool call outputs and can be used for parsing
                tool_calls = TypeAdapter(
                    list[FunctionDefinition]).validate_json(output.text)
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
                        tool_call_class(function=FunctionCall(
                            name=tool_call.name,
992
993
                            arguments=json.dumps(tool_call.parameters,
                                                 ensure_ascii=False)))
994
995
996
                        for tool_call in tool_calls
                    ])

997
998
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
999
            elif not request.tool_choice or request.tool_choice == "none":
1000

1001
1002
1003
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1004
1005
1006
1007
1008
1009
1010

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

1011
1012
1013
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
1014
                    logger.exception("Error in tool parser creation.")
1015
1016
                    return self.create_error_response(str(e))

1017
                tool_call_info = tool_parser.extract_tool_calls(
1018
                    content if content is not None else "", request=request)
1019
1020
1021
1022
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
1023
1024
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
1025
                                          reasoning_content=reasoning_content,
1026
1027
1028
1029
1030
1031
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1032
1033
1034
                    message = ChatMessage(role=role,
                                          reasoning_content=reasoning_content,
                                          content=content)
1035
1036
1037
1038
1039
1040
1041

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
1042
1043
1044
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1045

1046
1047
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1048
                message=message,
1049
                logprobs=logprobs,
1050
                finish_reason="tool_calls" if auto_tools_called else
1051
                output.finish_reason if output.finish_reason else "stop",
1052
                stop_reason=output.stop_reason)
1053
1054
            choices.append(choice_data)

1055
        if request.echo:
1056
            last_msg_content: Union[str, list[dict[str, str]]] = ""
1057
1058
            if conversation and "content" in conversation[-1] and conversation[
                    -1].get("role") == role:
1059
                last_msg_content = conversation[-1]["content"] or ""
1060
1061
1062
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
1063
1064

            for choice in choices:
1065
1066
                full_message = last_msg_content + (choice.message.content
                                                   or "")
1067
1068
                choice.message.content = full_message

1069
        assert final_res.prompt_token_ids is not None
1070
        num_prompt_tokens = len(final_res.prompt_token_ids)
1071
1072
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1073
1074
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
1075
1076
1077
1078
1079
1080
1081
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
1082
1083
1084

        request_metadata.final_usage_info = usage

1085
1086
1087
1088
1089
1090
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1091
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
Robert Shaw's avatar
Robert Shaw committed
1092
            kv_transfer_params=final_res.kv_transfer_params,
1093
1094
        )

1095
        return response
1096
1097

    def _get_top_logprobs(
1098
            self, logprobs: dict[int, Logprob], top_logprobs: Optional[int],
1099
1100
            tokenizer: AnyTokenizer,
            should_return_as_token_id: bool) -> list[ChatCompletionLogProb]:
1101
        return [
1102
1103
1104
1105
            ChatCompletionLogProb(token=(token := self._get_decoded_token(
                p[1],
                p[0],
                tokenizer,
1106
                return_as_token_id=should_return_as_token_id)),
1107
1108
1109
                                  logprob=max(p[1].logprob, -9999.0),
                                  bytes=list(
                                      token.encode("utf-8", errors="replace")))
1110
1111
1112
1113
1114
1115
1116
            for i, p in enumerate(logprobs.items())
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1117
        top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
1118
        tokenizer: AnyTokenizer,
1119
        num_output_top_logprobs: Optional[int] = None,
1120
        return_as_token_id: Optional[bool] = None,
1121
1122
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1123
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1124

1125
1126
        should_return_as_token_id = return_as_token_id if \
            return_as_token_id is not None else self.return_tokens_as_token_ids
1127
1128
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1129
1130
            if step_top_logprobs is None or step_top_logprobs.get(
                    token_id) is None:
1131
                token = tokenizer.decode(token_id)
1132
                if should_return_as_token_id:
1133
                    token = f"token_id:{token_id}"
1134

1135
1136
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1137
                        token=token,
1138
1139
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
1140
            else:
1141
1142
1143
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1144
1145
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1146
                        token=self._get_decoded_token(
1147
1148
1149
                            step_token,
                            token_id,
                            tokenizer,
1150
                            should_return_as_token_id,
1151
1152
1153
1154
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
1155
                        top_logprobs=self._get_top_logprobs(
1156
1157
                            step_top_logprobs, num_output_top_logprobs,
                            tokenizer, should_return_as_token_id),
1158
                    ))
1159
1160

        return ChatCompletionLogProbs(content=logprobs_content)
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1190
1191
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
1192
1193
1194
1195
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )