serving_chat.py 57.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
import time
7
8
9
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Callable, Final, Optional, Union
10

11
import jinja2
12
import partial_json_parser
13
import regex as re
14
from fastapi import Request
15
from pydantic import TypeAdapter
16

17
from vllm.config import ModelConfig
18
from vllm.engine.protocol import EngineClient
19
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
20
21
                                         ConversationMessage,
                                         random_tool_call_id)
22
from vllm.entrypoints.logger import RequestLogger
23
from vllm.entrypoints.openai.protocol import (
24
25
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
26
    ChatCompletionRequest, ChatCompletionResponse,
27
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
28
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
29
30
    DeltaToolCall, ErrorResponse, FunctionCall, FunctionDefinition,
    PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo)
31
32
from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
                                                    clamp_prompt_logprobs)
33
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
34
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
35
36
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
    MistralToolCall)
37
from vllm.entrypoints.utils import get_max_tokens
38
from vllm.logger import init_logger
39
from vllm.outputs import CompletionOutput, RequestOutput
40
from vllm.reasoning import ReasoningParser, ReasoningParserManager
41
from vllm.sampling_params import BeamSearchParams, SamplingParams
42
from vllm.sequence import Logprob
43
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
44
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
45
46
                                                truncate_tool_call_ids,
                                                validate_request_params)
47
48
49
50
51
52

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

53
54
55
56
    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
57
        models: OpenAIServingModels,
58
59
60
61
62
63
        response_role: str,
        *,
        request_logger: Optional[RequestLogger],
        chat_template: Optional[str],
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
64
        reasoning_parser: str = "",
65
66
67
        enable_auto_tools: bool = False,
        tool_parser: Optional[str] = None,
        enable_prompt_tokens_details: bool = False,
68
        enable_force_include_usage: bool = False,
69
    ) -> None:
70
        super().__init__(engine_client=engine_client,
71
                         model_config=model_config,
72
                         models=models,
73
                         request_logger=request_logger,
74
75
                         return_tokens_as_token_ids=return_tokens_as_token_ids,
                         enable_force_include_usage=enable_force_include_usage)
76

77
        self.response_role = response_role
78
79
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
80

81
82
83
84
85
86
87
88
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

89
90
        self.reasoning_parser: Optional[Callable[[AnyTokenizer],
                                                 ReasoningParser]] = None
91
        if reasoning_parser:
92
93
94
95
            try:
                self.reasoning_parser = (
                    ReasoningParserManager.get_reasoning_parser(
                        reasoning_parser))
96
                assert self.reasoning_parser is not None
97
            except Exception as e:
98
99
                raise TypeError(
                    f"{reasoning_parser=} has not been registered") from e
100
101
        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
102
            try:
103
104
105
106
107
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
108
109
110
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
111
                raise TypeError("Error: --enable-auto-tool-choice requires "
112
113
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
114

115
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
116
        self.enable_force_include_usage = enable_force_include_usage
117
118
119
        self.default_sampling_params = (
            self.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
120
121
122
123
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
            logger.info("Using default chat sampling params from %s: %s",
                        source, self.default_sampling_params)
124

125
    async def create_chat_completion(
126
127
        self,
        request: ChatCompletionRequest,
128
129
130
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
131
132
        """
        Chat Completion API similar to OpenAI's API.
133

134
135
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
136
        Chat Completion API.
137
138
139
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
140
            logger.error("Error with model %s", error_check_ret)
141
142
            return error_check_ret

143
144
145
146
147
148
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

149
        try:
150
151
            lora_request = self._maybe_get_adapters(
                request, supports_default_mm_loras=True)
152

153
            model_name = self._get_model_name(request.model, lora_request)
154

155
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
156

157
158
            tool_parser = self.tool_parser

159
            if isinstance(tokenizer, MistralTokenizer):
160
161
162
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
163
                maybe_serialize_tool_calls(request)
164
                truncate_tool_call_ids(request)
165
                validate_request_params(request)
166

167
168
169
170
171
172
173
174
175
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
                    and not isinstance(tokenizer, MistralTokenizer)):
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
176

177
178
179
180
            if request.tools is None:
                tool_dicts = None
            else:
                tool_dicts = [tool.model_dump() for tool in request.tools]
181

182
183
184
185
186
187
188
189
190
            (
                conversation,
                request_prompts,
                engine_prompts,
            ) = await self._preprocess_chat(
                request,
                tokenizer,
                request.messages,
                chat_template=request.chat_template or self.chat_template,
191
                chat_template_content_format=self.chat_template_content_format,
192
193
194
195
196
197
198
199
200
                add_generation_prompt=request.add_generation_prompt,
                continue_final_message=request.continue_final_message,
                tool_dicts=tool_dicts,
                documents=request.documents,
                chat_template_kwargs=request.chat_template_kwargs,
                tool_parser=tool_parser,
                truncate_prompt_tokens=request.truncate_prompt_tokens,
                add_special_tokens=request.add_special_tokens,
            )
201
202
        except (ValueError, TypeError, RuntimeError,
                jinja2.TemplateError) as e:
203
            logger.exception("Error in preprocessing prompt inputs")
204
            return self.create_error_response(f"{e} {e.__cause__}")
205

206
207
        request_id = "chatcmpl-" \
                     f"{self._base_request_id(raw_request, request.request_id)}"
208
209
210
211
212

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

213
        # Schedule the request and get the result generator.
214
        generators: list[AsyncGenerator[RequestOutput, None]] = []
215
        try:
216
217
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
218
219
220
221
222
223
224
225
226
227

                if self.default_sampling_params is None:
                    self.default_sampling_params = {}

                max_tokens = get_max_tokens(
                    max_model_len=self.max_model_len,
                    request=request,
                    input_length=len(engine_prompt["prompt_token_ids"]),
                    default_sampling_params=self.default_sampling_params)

228
229
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
230
                        max_tokens, self.default_sampling_params)
231
232
                else:
                    sampling_params = request.to_sampling_params(
233
                        max_tokens, self.model_config.logits_processor_pattern,
234
                        self.default_sampling_params)
235
236
237
238

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
239
                                 lora_request=lora_request)
240
241
242
243
244
245
246
247
248

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
249
                        lora_request=lora_request,
250
251
252
253
254
255
256
257
258
259
260
261
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        priority=request.priority,
                    )

                generators.append(generator)
262
        except ValueError as e:
263
            # TODO: Use a vllm-specific Validation Error
264
265
            return self.create_error_response(str(e))

266
267
268
        assert len(generators) == 1
        result_generator, = generators

269
270
271
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
272
273
274
275
276
277
278
279
                request,
                result_generator,
                request_id,
                model_name,
                conversation,
                tokenizer,
                request_metadata,
                enable_force_include_usage=self.enable_force_include_usage)
280

281
282
        try:
            return await self.chat_completion_full_generator(
283
284
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
285
286
287
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
288
289
290
291

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
292
        return request.messages[-1]["role"]
293

294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    @staticmethod
    def _bracket_level(s: str, opening='{', closing='}') -> int:
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
    def _filter_delta_text(delta_text: str,
                           previous_text: str) -> tuple[str, bool]:
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
            if c == '{':
                bracket_level += 1
                passed_zero = bracket_level == 0
            elif c == '}':
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
                if c == ',':
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
337
        current_text: Optional[str],
338
339
340
        delta_text: str,
        function_name_returned: bool,
    ) -> tuple[Optional[DeltaMessage], bool]:
341
342
343
        if current_text is None or current_text == "":
            # if the current text is empty, we cannot parse it
            return None, function_name_returned
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
            logger.debug('not enough tokens to parse into JSON yet')
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
                delta_text, previous_text)
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
            if not finishes_previous_tool and ("name" not in current_tool_call
                                               or "parameters"
                                               not in current_tool_call):
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
                    param_match = re.search(r'.*"parameters":\s*(.*)',
                                            current_text)
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
                        arguments, previous_text)

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
                    if (finishes_previous_tool
                            and "parameters" not in current_tool_call):
                        current_tool_call = obj[-2]

                    function_name_returned = True
                    delta_message = DeltaMessage(tool_calls=[
386
387
388
389
                        DeltaToolCall(id=random_tool_call_id(),
                                      function=DeltaFunctionCall(
                                          name=current_tool_call["name"],
                                          arguments=arguments),
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
                                      index=len(obj) - 1,
                                      type="function")
                    ])

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
                        delta_text, previous_text)

                    if delta_text != "":
                        delta_message = DeltaMessage(tool_calls=[
                            DeltaToolCall(
                                function=DeltaFunctionCall(
                                    # OpenAI API returns None
                                    # instead of name every time
                                    name=None,
                                    arguments=delta_text),
406
                                index=len(obj) - 1)
407
408
409
410
411
412
                        ])
                    else:
                        delta_message = None

        return delta_message, function_name_returned

413
    async def chat_completion_stream_generator(
414
415
416
417
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
418
        model_name: str,
419
        conversation: list[ConversationMessage],
420
        tokenizer: AnyTokenizer,
421
        request_metadata: RequestResponseMetadata,
422
        enable_force_include_usage: bool,
423
    ) -> AsyncGenerator[str, None]:
424
        created_time = int(time.time())
425
        chunk_object_type: Final = "chat.completion.chunk"
426
        first_iteration = True
427
428

        # Send response for each token for each request.n (index)
429
430
431
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
432
        num_prompt_tokens = 0
433
        num_cached_tokens = None
434
435
436
437
438
439
440
441
442
443
444

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

445
        all_previous_token_ids: Optional[list[list[int]]]
446
        function_name_returned = [False] * num_choices
447
448
449

        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
450
        if tool_choice_auto or self.reasoning_parser:
451
452
453
            # These are only required in "auto" tool choice case
            previous_texts = [""] * num_choices
            all_previous_token_ids = [[]] * num_choices
454
455
456
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
457
458
459
        elif request.tool_choice == "required":
            previous_texts = [""] * num_choices
            all_previous_token_ids = None
460
461
462
        else:
            previous_texts, all_previous_token_ids = None, None

463
        try:
464
            if self.reasoning_parser:
465
466
467
468
469
470
471
                reasoning_parser = self.reasoning_parser(tokenizer)
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
472
473
474
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
475
                tool_parsers: list[Optional[ToolParser]] = [
476
477
478
479
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
480
        except Exception as e:
481
            logger.exception("Error in tool parser creation.")
482
483
484
485
486
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

487
488
        stream_options = request.stream_options
        if stream_options:
489
490
            include_usage = stream_options.include_usage \
                            or enable_force_include_usage
491
492
493
494
495
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

496
497
        try:
            async for res in result_generator:
498
499
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
500
501
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
502

503
504
505
506
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
507
                    num_cached_tokens = res.num_cached_tokens
508
509
                    # Send first response for each request.n (index) with
                    # the role
510
                    role = self.get_chat_request_role(request)
511
512
513

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
514
                    for i in range(num_choices):
515
516
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
517
518
519
520
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
521
522
523
524
525
526
527
528
                            logprobs=None,
                            finish_reason=None)
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
                            model=model_name)
529

530
531
532
533
534
535
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
536

537
538
539
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

540
541
                    # Send response to echo the input portion of the
                    # last message
542
                    if request.echo:
543
                        last_msg_content: Union[str, list[dict[str, str]]] = ""
544
545
546
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
547
548

                        if last_msg_content:
549
                            for i in range(num_choices):
550
551
552
553
554
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
555
                                        logprobs=None,
556
                                        finish_reason=None))
557
558
559
560
561
562
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
563
564
565
566
567
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
568

569
570
571
572
573
574
575
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
576
                    tool_parser = tool_parsers[i]
577
578
579
580

                    if finish_reason_sent[i]:
                        continue

581
                    if request.logprobs and request.top_logprobs is not None:
582
                        assert output.logprobs is not None, (
583
                            "Did not output logprobs")
584
                        logprobs = self._create_chat_logprobs(
585
586
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
587
                            tokenizer=tokenizer,
588
                            num_output_top_logprobs=request.top_logprobs,
589
590
                            return_as_token_id=request.
                            return_tokens_as_token_ids,
591
592
593
594
                        )
                    else:
                        logprobs = None

595
                    delta_text = output.text
596
597
598
599
600
601

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

602
                    delta_message: Optional[DeltaMessage]
603

604
                    # just update previous_texts and previous_token_ids
605
                    if tool_choice_auto or self.reasoning_parser:
606
607
608
609
610
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
611
612
613
614
615
616
617

                        # avoid the None + list error.
                        if previous_token_ids:
                            current_token_ids = previous_token_ids + list(
                                output.token_ids)
                        else:
                            current_token_ids = list(output.token_ids)
618

619
620
                    # handle streaming deltas for tools with named tool_choice
                    if tool_choice_function_name:
621
                        if (self.reasoning_parser
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
                                and not reasoning_parser.is_reasoning_end(
                                    previous_token_ids)):
                            assert reasoning_parser is not None
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))
                            # When encountering think end id in delta_token_ids,
                            # process the `content`. Only keep 'content',
                            # remove 'reasoning_content'
                            if reasoning_parser.is_reasoning_end(
                                    list(output.token_ids)):
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
648
                            if self.reasoning_parser:
649
650
651
                                delta_text = previous_text + delta_text
                                current_text = ""

652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        arguments=delta_text),
                                    index=i)
                            else:
                                delta_tool_call = DeltaToolCall(
                                    id=random_tool_call_id(),
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
                                        arguments=delta_text),
                                    index=i)
                                function_name_returned[i] = True

667
                            delta_message = DeltaMessage(tool_calls=[
668
                                delta_tool_call,
669
670
                            ])

671
672
673
674
675
676
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]

677
678
679
680
681
682
683
684
                        if self.reasoning_parser:
                            _, content = \
                                reasoning_parser.extract_reasoning_content(
                                    current_text,
                                    request
                                )
                        else:
                            content = current_text
685
686
687
                        delta_message, function_name_returned[i] = (
                            self.extract_tool_call_required_streaming(
                                previous_text=previous_text,
688
                                current_text=content,
689
690
691
692
693
694
                                delta_text=delta_text,
                                function_name_returned=fn_name_returned))

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text

695
696
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
697
                    elif tool_choice_auto and self.reasoning_parser:
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
                        if not reasoning_end_arr[i]:
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
                            # When encountering think end id in prompt_token_ids
                            # i.e {"enable_thinking": False},
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if res.prompt_token_ids and \
                                reasoning_parser.is_reasoning_end(
                                    list(res.prompt_token_ids)):
                                reasoning_end_arr[i] = True
                                current_token_ids = list(output.token_ids)
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if reasoning_parser.is_reasoning_end(
                                    list(output.token_ids)):
                                reasoning_end_arr[i] = True
                                current_token_ids =  \
                                    reasoning_parser.extract_content_ids(
                                        list(output.token_ids))
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
                            delta_token_ids = list(output.token_ids)
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

                            delta_message = (
                                tool_parser.extract_tool_calls_streaming(
                                    previous_text=previous_text,
                                    current_text=current_text,
                                    delta_text=delta_text,
                                    previous_token_ids=previous_token_ids,
                                    current_token_ids=current_token_ids,
                                    delta_token_ids=delta_token_ids,
                                    request=request))
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
769
770
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
771
772
                                previous_text=previous_text,
                                current_text=current_text,
773
                                delta_text=delta_text,
774
775
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
776
777
                                delta_token_ids=output.token_ids,
                                request=request))
778
                    # when only reasoning
779
                    elif self.reasoning_parser:
780
781
782
783
784
785
786
787
788
                        delta_message = (reasoning_parser.
                                         extract_reasoning_content_streaming(
                                             previous_text,
                                             current_text,
                                             delta_text,
                                             previous_token_ids,
                                             current_token_ids,
                                             output.token_ids,
                                         ))
789
                    # handle streaming just a content delta
790
791
792
                    else:
                        delta_message = DeltaMessage(content=delta_text)

793
                    # update the previous values for the next iteration
794
                    if tool_choice_auto or self.reasoning_parser:
795
796
797
798
799
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids

800
                    # set the previous values for the next iteration
801
                    previous_num_tokens[i] += len(output.token_ids)
802
803
804
805
806
807
808
809

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

810
811
812
813
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
814
                            delta=delta_message,
815
816
                            logprobs=logprobs,
                            finish_reason=None)
817
818

                    # if the model is finished generating
819
                    else:
820
821
822
823
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
824
                        auto_tools_called = False
825
                        if tool_parser:
826
827
828
829
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
830
831
832
833
834
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
835
836
837
838
839
840
841
842
843
844
                            latest_delta_len = 0
                            if ((isinstance(
                                    delta_message.tool_calls[0].function,
                                    DeltaFunctionCall)) and isinstance(
                                        delta_message.tool_calls[0].function.
                                        arguments, str)):
                                latest_delta_len = len(
                                    delta_message.tool_calls[0].function.
                                    arguments)

845
846
847
848
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
849
850
                                    "arguments", {}),
                                ensure_ascii=False)
851

852
                            # get what we've streamed so far for arguments
853
854
855
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]
856
857
                            if (latest_delta_len > 0):
                                actual_call = actual_call[:-latest_delta_len]
858
859
860
861
862
863
864
865
866
867
868
869

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)
                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

870
871
872
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
873
                            delta=delta_message,
874
                            logprobs=logprobs,
875
                            finish_reason=output.finish_reason
876
                            if not auto_tools_called else "tool_calls",
877
                            stop_reason=output.stop_reason)
878

879
                        finish_reason_sent[i] = True
880

881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

897
                    data = chunk.model_dump_json(exclude_unset=True)
898
899
                    yield f"data: {data}\n\n"

900
901
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
902
903
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
904
905
906
907
908
909
910
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
911
912
913
914
915
916
917
918
919
920
921

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
922

923
924
925
926
927
928
929
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
                total_tokens=num_prompt_tokens + num_completion_tokens)

930
        except Exception as e:
931
            # TODO: Use a vllm-specific Validation Error
932
            logger.exception("Error in chat completion stream generator.")
933
934
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
935
936
937
938
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
939
940
941
942
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
943
        model_name: str,
944
        conversation: list[ConversationMessage],
945
        tokenizer: AnyTokenizer,
946
        request_metadata: RequestResponseMetadata,
947
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
948

949
        created_time = int(time.time())
950
        final_res: Optional[RequestOutput] = None
951

952
953
954
955
956
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
957
958
959
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
960

961
962
        assert final_res is not None

963
        choices: list[ChatCompletionResponseChoice] = []
964

965
966
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
967
            token_ids = output.token_ids
968
            out_logprobs = output.logprobs
969

970
971
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
972
                logprobs = self._create_chat_logprobs(
973
                    token_ids=token_ids,
974
                    top_logprobs=out_logprobs,
975
                    num_output_top_logprobs=request.top_logprobs,
976
                    tokenizer=tokenizer,
977
                    return_as_token_id=request.return_tokens_as_token_ids,
978
979
980
                )
            else:
                logprobs = None
981
            auto_tools_called = False
982

983
            if self.reasoning_parser:
984
985
986
987
988
                try:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
989
990
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
991
992
993
                reasoning_content, content = (
                    reasoning_parser.extract_reasoning_content(
                        output.text, request=request))
994
995
996
            else:
                reasoning_content = None
                content = output.text
997

998
999
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
1000
1001
1002
1003
            if (not self.enable_auto_tools or not self.tool_parser) and \
                (not isinstance(request.tool_choice,
                                ChatCompletionNamedToolChoiceParam
                                ) and request.tool_choice != "required"):
1004
1005
1006
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1007
1008
1009

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
1010
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
1011

1012
1013
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall
1014
1015
                message = ChatMessage(
                    role=role,
1016
                    reasoning_content=reasoning_content,
1017
1018
                    content="",
                    tool_calls=[
1019
                        tool_call_class(function=FunctionCall(
1020
                            name=request.tool_choice.function.name,
1021
                            arguments=content))
1022
                    ])
1023

1024
1025
1026
1027
1028
1029
            elif request.tool_choice and request.tool_choice == "required":
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall

                # the fields of FunctionDefinition are a superset of the
                # tool call outputs and can be used for parsing
1030
                assert content is not None
1031
                tool_calls = TypeAdapter(
1032
                    list[FunctionDefinition]).validate_json(content)
1033
1034
1035
                message = ChatMessage(
                    role=role,
                    content="",
1036
                    reasoning_content=reasoning_content,
1037
1038
1039
                    tool_calls=[
                        tool_call_class(function=FunctionCall(
                            name=tool_call.name,
1040
1041
                            arguments=json.dumps(tool_call.parameters,
                                                 ensure_ascii=False)))
1042
1043
1044
                        for tool_call in tool_calls
                    ])

1045
1046
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
1047
            elif not request.tool_choice or request.tool_choice == "none":
1048

1049
1050
1051
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1052
1053
1054
1055
1056
1057
1058

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

1059
1060
1061
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
1062
                    logger.exception("Error in tool parser creation.")
1063
1064
                    return self.create_error_response(str(e))

1065
                tool_call_info = tool_parser.extract_tool_calls(
1066
                    content if content is not None else "", request=request)
1067
1068
1069
1070
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
1071
1072
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
1073
                                          reasoning_content=reasoning_content,
1074
1075
1076
1077
1078
1079
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1080
1081
1082
1083
1084
1085
1086
1087
                    ret_content = content

                    # try to use content return from tool parser first,
                    # tool parser may do some modify for the content.
                    if (tool_call_info.content
                            and len(tool_call_info.content) > 0):
                        ret_content = tool_call_info.content

1088
1089
                    message = ChatMessage(role=role,
                                          reasoning_content=reasoning_content,
1090
                                          content=ret_content)
1091
1092
1093
1094
1095
1096
1097

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
1098
1099
1100
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1101

1102
1103
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1104
                message=message,
1105
                logprobs=logprobs,
1106
                finish_reason="tool_calls" if auto_tools_called else
1107
                output.finish_reason if output.finish_reason else "stop",
1108
                stop_reason=output.stop_reason)
1109
1110
            choices.append(choice_data)

1111
        if request.echo:
1112
            last_msg_content: Union[str, list[dict[str, str]]] = ""
1113
1114
            if conversation and "content" in conversation[-1] and conversation[
                    -1].get("role") == role:
1115
                last_msg_content = conversation[-1]["content"] or ""
1116
1117
1118
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
1119
1120

            for choice in choices:
1121
1122
                full_message = last_msg_content + (choice.message.content
                                                   or "")
1123
1124
                choice.message.content = full_message

1125
        assert final_res.prompt_token_ids is not None
1126
        num_prompt_tokens = len(final_res.prompt_token_ids)
1127
1128
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1129
1130
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
1131
1132
1133
1134
1135
1136
1137
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
1138
1139
1140

        request_metadata.final_usage_info = usage

1141
1142
1143
1144
1145
1146
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1147
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
Robert Shaw's avatar
Robert Shaw committed
1148
            kv_transfer_params=final_res.kv_transfer_params,
1149
1150
        )

1151
        return response
1152
1153

    def _get_top_logprobs(
1154
            self, logprobs: dict[int, Logprob], top_logprobs: Optional[int],
1155
1156
            tokenizer: AnyTokenizer,
            should_return_as_token_id: bool) -> list[ChatCompletionLogProb]:
1157
        return [
1158
1159
1160
1161
            ChatCompletionLogProb(token=(token := self._get_decoded_token(
                p[1],
                p[0],
                tokenizer,
1162
                return_as_token_id=should_return_as_token_id)),
1163
1164
1165
                                  logprob=max(p[1].logprob, -9999.0),
                                  bytes=list(
                                      token.encode("utf-8", errors="replace")))
1166
1167
1168
1169
1170
1171
1172
            for i, p in enumerate(logprobs.items())
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1173
        top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
1174
        tokenizer: AnyTokenizer,
1175
        num_output_top_logprobs: Optional[int] = None,
1176
        return_as_token_id: Optional[bool] = None,
1177
1178
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1179
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1180

1181
1182
        should_return_as_token_id = return_as_token_id if \
            return_as_token_id is not None else self.return_tokens_as_token_ids
1183
1184
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1185
1186
            if step_top_logprobs is None or step_top_logprobs.get(
                    token_id) is None:
1187
                token = tokenizer.decode(token_id)
1188
                if should_return_as_token_id:
1189
                    token = f"token_id:{token_id}"
1190

1191
1192
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1193
                        token=token,
1194
1195
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
1196
            else:
1197
1198
1199
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1200
1201
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1202
                        token=self._get_decoded_token(
1203
1204
1205
                            step_token,
                            token_id,
                            tokenizer,
1206
                            should_return_as_token_id,
1207
1208
1209
1210
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
1211
                        top_logprobs=self._get_top_logprobs(
1212
1213
                            step_top_logprobs, num_output_top_logprobs,
                            tokenizer, should_return_as_token_id),
1214
                    ))
1215
1216

        return ChatCompletionLogProbs(content=logprobs_content)
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1246
1247
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
1248
1249
1250
1251
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )