"examples/other/fp8/README.md" did not exist on "5950f555a1d2ce19c30efb24abe03737320d05c1"
serving_chat.py 54.8 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import asyncio
4
import json
5
import time
6
7
8
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import Callable, Final, Optional, Union
9

10
import jinja2
11
import partial_json_parser
12
import regex as re
13
from fastapi import Request
14
from pydantic import TypeAdapter
15

16
from vllm.config import ModelConfig
17
from vllm.engine.protocol import EngineClient
18
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
19
20
                                         ConversationMessage,
                                         random_tool_call_id)
21
from vllm.entrypoints.logger import RequestLogger
22
from vllm.entrypoints.openai.protocol import (
23
24
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
25
    ChatCompletionRequest, ChatCompletionResponse,
26
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
27
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
28
29
    DeltaToolCall, ErrorResponse, FunctionCall, FunctionDefinition,
    PromptTokenUsageInfo, RequestResponseMetadata, ToolCall, UsageInfo)
30
31
from vllm.entrypoints.openai.serving_engine import (OpenAIServing,
                                                    clamp_prompt_logprobs)
32
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
33
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
34
35
from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import (
    MistralToolCall)
36
from vllm.logger import init_logger
37
from vllm.outputs import CompletionOutput, RequestOutput
38
from vllm.reasoning import ReasoningParser, ReasoningParserManager
39
from vllm.sampling_params import BeamSearchParams, SamplingParams
40
from vllm.sequence import Logprob
41
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
42
from vllm.transformers_utils.tokenizers import (maybe_serialize_tool_calls,
43
44
                                                truncate_tool_call_ids,
                                                validate_request_params)
45
46
47
48
49
50

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

51
52
53
54
    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
55
        models: OpenAIServingModels,
56
57
58
59
60
61
        response_role: str,
        *,
        request_logger: Optional[RequestLogger],
        chat_template: Optional[str],
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
62
        reasoning_parser: str = "",
63
64
65
66
        enable_auto_tools: bool = False,
        tool_parser: Optional[str] = None,
        enable_prompt_tokens_details: bool = False,
    ) -> None:
67
        super().__init__(engine_client=engine_client,
68
                         model_config=model_config,
69
                         models=models,
70
71
                         request_logger=request_logger,
                         return_tokens_as_token_ids=return_tokens_as_token_ids)
72

73
        self.response_role = response_role
74
75
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
76

77
78
79
80
81
82
83
84
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

85
86
        self.reasoning_parser: Optional[Callable[[AnyTokenizer],
                                                 ReasoningParser]] = None
87
        if reasoning_parser:
88
89
90
91
            try:
                self.reasoning_parser = (
                    ReasoningParserManager.get_reasoning_parser(
                        reasoning_parser))
92
                assert self.reasoning_parser is not None
93
            except Exception as e:
94
95
                raise TypeError(
                    f"{reasoning_parser=} has not been registered") from e
96
97
        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
98
            try:
99
100
101
102
103
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
104
105
106
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
107
                raise TypeError("Error: --enable-auto-tool-choice requires "
108
109
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
110

111
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
112
113
114
        self.default_sampling_params = (
            self.model_config.get_diff_sampling_param())
        if self.default_sampling_params:
115
116
117
118
            source = self.model_config.generation_config
            source = "model" if source == "auto" else source
            logger.info("Using default chat sampling params from %s: %s",
                        source, self.default_sampling_params)
119

120
    async def create_chat_completion(
121
122
        self,
        request: ChatCompletionRequest,
123
124
125
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
126
127
        """
        Chat Completion API similar to OpenAI's API.
128

129
130
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
131
        Chat Completion API.
132
133
134
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
135
            logger.error("Error with model %s", error_check_ret)
136
137
            return error_check_ret

138
139
140
141
142
143
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

144
        try:
145
146
147
148
149
            (
                lora_request,
                prompt_adapter_request,
            ) = self._maybe_get_adapters(request)

150
            model_name = self._get_model_name(request.model, lora_request)
151

152
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
153

154
155
            tool_parser = self.tool_parser

156
            if isinstance(tokenizer, MistralTokenizer):
157
158
159
                # because of issues with pydantic we need to potentially
                # re-serialize the tool_calls field of the request
                # for more info: see comment in `maybe_serialize_tool_calls`
160
                maybe_serialize_tool_calls(request)
161
                truncate_tool_call_ids(request)
162
                validate_request_params(request)
163

164
165
166
167
168
169
170
171
172
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
                    and not isinstance(tokenizer, MistralTokenizer)):
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
173

174
175
176
177
            tool_dicts = None if request.tools is None else [
                tool.model_dump() for tool in request.tools
            ]

178
179
180
181
182
183
184
185
186
            (
                conversation,
                request_prompts,
                engine_prompts,
            ) = await self._preprocess_chat(
                request,
                tokenizer,
                request.messages,
                chat_template=request.chat_template or self.chat_template,
187
                chat_template_content_format=self.chat_template_content_format,
188
189
190
191
192
193
194
195
196
                add_generation_prompt=request.add_generation_prompt,
                continue_final_message=request.continue_final_message,
                tool_dicts=tool_dicts,
                documents=request.documents,
                chat_template_kwargs=request.chat_template_kwargs,
                tool_parser=tool_parser,
                truncate_prompt_tokens=request.truncate_prompt_tokens,
                add_special_tokens=request.add_special_tokens,
            )
197
198
        except (ValueError, TypeError, RuntimeError,
                jinja2.TemplateError) as e:
199
            logger.exception("Error in preprocessing prompt inputs")
200
            return self.create_error_response(f"{e} {e.__cause__}")
201

202
203
        request_id = "chatcmpl-" \
                     f"{self._base_request_id(raw_request, request.request_id)}"
204
205
206
207
208

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

209
        # Schedule the request and get the result generator.
210
        generators: list[AsyncGenerator[RequestOutput, None]] = []
211
        try:
212
213
214
215
216
217
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
                default_max_tokens = self.max_model_len - len(
                    engine_prompt["prompt_token_ids"])
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
218
                        default_max_tokens, self.default_sampling_params)
219
220
                else:
                    sampling_params = request.to_sampling_params(
221
                        default_max_tokens,
222
                        self.model_config.logits_processor_pattern,
223
                        self.default_sampling_params)
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
                                 lora_request=lora_request,
                                 prompt_adapter_request=prompt_adapter_request)

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
239
                        lora_request=lora_request,
240
241
242
243
244
245
246
247
248
249
250
251
252
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        prompt_adapter_request=prompt_adapter_request,
                        priority=request.priority,
                    )

                generators.append(generator)
253
        except ValueError as e:
254
            # TODO: Use a vllm-specific Validation Error
255
256
            return self.create_error_response(str(e))

257
258
259
        assert len(generators) == 1
        result_generator, = generators

260
261
262
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
263
264
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
265

266
267
        try:
            return await self.chat_completion_full_generator(
268
269
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
270
271
272
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
273
274
275
276

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
277
        return request.messages[-1]["role"]
278

279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
    @staticmethod
    def _bracket_level(s: str, opening='{', closing='}') -> int:
        """
        Calculate the current level of nested brackets in a given string.
        """
        level = 0
        for char in s:
            if char == opening:
                level += 1
            elif char == closing:
                level -= 1
        return level

    @staticmethod
    def _filter_delta_text(delta_text: str,
                           previous_text: str) -> tuple[str, bool]:
        # remove last '},' of the tool definition stemming from the
        # "name"/"parameters" outer object or closing ']' of the tool list
        # count occurrences of opening and closing curly braces and
        # once level 0 is reached stop outputting text
        # if 0 is reached while parsing the delta_text we know the current
        # tool will finish in this current iteration
        bracket_level = OpenAIServingChat._bracket_level(previous_text)
        updated_delta, passed_zero = "", False
        for c in delta_text:
            if c == '{':
                bracket_level += 1
                passed_zero = bracket_level == 0
            elif c == '}':
                bracket_level -= 1
                passed_zero = bracket_level == 0

            if bracket_level != 0:
                updated_delta += c
            else:
                # if a comma is reached at level 0 we can stop
                if c == ',':
                    break
        return updated_delta, passed_zero

    def extract_tool_call_required_streaming(
        self,
        previous_text: str,
        current_text: str,
        delta_text: str,
        function_name_returned: bool,
    ) -> tuple[Optional[DeltaMessage], bool]:
        try:
            obj = partial_json_parser.loads(current_text)
        except partial_json_parser.core.exceptions.MalformedJSON:
            logger.debug('not enough tokens to parse into JSON yet')
            obj = None

        # check if the current text is a valid array
        # containing a partial tool calling object
        # if not repeat
        if obj is None or not isinstance(obj, list) or not len(obj) > 0:
            function_name_returned = False
            delta_message = None
        else:
            _, finishes_previous_tool = OpenAIServingChat._filter_delta_text(
                delta_text, previous_text)
            # take the last tool call from the generated list
            current_tool_call = obj[-1]

            # once parameters have been generated the name is complete as well
            if not finishes_previous_tool and ("name" not in current_tool_call
                                               or "parameters"
                                               not in current_tool_call):
                function_name_returned = False
                delta_message = None
            else:
                if not function_name_returned:
                    # get partly generated arguments from the latest tool call
                    param_match = re.search(r'.*"parameters":\s*(.*)',
                                            current_text)
                    arguments = param_match.group(1) if param_match else ""
                    arguments, _ = OpenAIServingChat._filter_delta_text(
                        arguments, previous_text)

                    # if this iteration finishes a previous tool call but a
                    # new incomplete tool is already generated, take the
                    # previous from the list
                    if (finishes_previous_tool
                            and "parameters" not in current_tool_call):
                        current_tool_call = obj[-2]

                    function_name_returned = True
                    delta_message = DeltaMessage(tool_calls=[
368
369
370
371
                        DeltaToolCall(id=random_tool_call_id(),
                                      function=DeltaFunctionCall(
                                          name=current_tool_call["name"],
                                          arguments=arguments),
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
                                      index=len(obj) - 1,
                                      type="function")
                    ])

                else:
                    delta_text, _ = OpenAIServingChat._filter_delta_text(
                        delta_text, previous_text)

                    if delta_text != "":
                        delta_message = DeltaMessage(tool_calls=[
                            DeltaToolCall(
                                function=DeltaFunctionCall(
                                    # OpenAI API returns None
                                    # instead of name every time
                                    name=None,
                                    arguments=delta_text),
388
                                index=len(obj) - 1)
389
390
391
392
393
394
                        ])
                    else:
                        delta_message = None

        return delta_message, function_name_returned

395
    async def chat_completion_stream_generator(
396
397
398
399
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
400
        model_name: str,
401
        conversation: list[ConversationMessage],
402
        tokenizer: AnyTokenizer,
403
        request_metadata: RequestResponseMetadata,
404
    ) -> AsyncGenerator[str, None]:
405
        created_time = int(time.time())
406
        chunk_object_type: Final = "chat.completion.chunk"
407
        first_iteration = True
408
409

        # Send response for each token for each request.n (index)
410
411
412
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
413
        num_prompt_tokens = 0
414
        num_cached_tokens = None
415
416
417
418
419
420
421
422
423
424
425

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

426
        all_previous_token_ids: Optional[list[list[int]]]
427
        function_name_returned = [False] * num_choices
428
429
430

        # Only one of these will be used, thus previous_texts and
        # all_previous_token_ids will not be used twice in the same iteration.
431
        if tool_choice_auto or self.reasoning_parser:
432
433
434
            # These are only required in "auto" tool choice case
            previous_texts = [""] * num_choices
            all_previous_token_ids = [[]] * num_choices
435
436
437
            # For reasoning parser and tool call all enabled
            added_content_delta_arr = [False] * num_choices
            reasoning_end_arr = [False] * num_choices
438
439
440
        elif request.tool_choice == "required":
            previous_texts = [""] * num_choices
            all_previous_token_ids = None
441
442
443
        else:
            previous_texts, all_previous_token_ids = None, None

444
        try:
445
            if self.reasoning_parser:
446
447
448
449
450
451
452
                reasoning_parser = self.reasoning_parser(tokenizer)
        except RuntimeError as e:
            logger.exception("Error in reasoning parser creation.")
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return
453
454
455
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
456
                tool_parsers: list[Optional[ToolParser]] = [
457
458
459
460
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
461
        except Exception as e:
462
            logger.exception("Error in tool parser creation.")
463
464
465
466
467
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

468
469
470
471
472
473
474
475
        stream_options = request.stream_options
        if stream_options:
            include_usage = stream_options.include_usage
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

476
477
        try:
            async for res in result_generator:
478
479
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
480
481
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
482

483
484
485
486
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
487
                    num_cached_tokens = res.num_cached_tokens
488
489
                    # Send first response for each request.n (index) with
                    # the role
490
                    role = self.get_chat_request_role(request)
491
492
493

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
494
                    for i in range(num_choices):
495
496
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
497
498
499
500
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
501
502
503
504
505
506
507
508
                            logprobs=None,
                            finish_reason=None)
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
                            model=model_name)
509

510
511
512
513
514
515
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
516

517
518
519
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

520
521
                    # Send response to echo the input portion of the
                    # last message
522
                    if request.echo:
523
                        last_msg_content: Union[str, list[dict[str, str]]] = ""
524
525
526
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
527
528

                        if last_msg_content:
529
                            for i in range(num_choices):
530
531
532
533
534
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
535
                                        logprobs=None,
536
                                        finish_reason=None))
537
538
539
540
541
542
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
543
544
545
546
547
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
548

549
550
551
552
553
554
555
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
556
                    tool_parser = tool_parsers[i]
557
558
559
560

                    if finish_reason_sent[i]:
                        continue

561
                    if request.logprobs and request.top_logprobs is not None:
562
                        assert output.logprobs is not None, (
563
                            "Did not output logprobs")
564
                        logprobs = self._create_chat_logprobs(
565
566
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
567
                            tokenizer=tokenizer,
568
                            num_output_top_logprobs=request.top_logprobs,
569
570
                            return_as_token_id=request.
                            return_tokens_as_token_ids,
571
572
573
574
                        )
                    else:
                        logprobs = None

575
                    delta_text = output.text
576
577
578
579
580
581

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

582
                    delta_message: Optional[DeltaMessage]
583

584
                    # just update previous_texts and previous_token_ids
585
                    if tool_choice_auto or self.reasoning_parser:
586
587
588
589
590
591
592
593
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
                        current_token_ids = previous_token_ids + list(
                            output.token_ids)

594
595
                    # handle streaming deltas for tools with named tool_choice
                    if tool_choice_function_name:
596
                        if (self.reasoning_parser
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
                                and not reasoning_parser.is_reasoning_end(
                                    previous_token_ids)):
                            assert reasoning_parser is not None
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))
                            # When encountering think end id in delta_token_ids,
                            # process the `content`. Only keep 'content',
                            # remove 'reasoning_content'
                            if reasoning_parser.is_reasoning_end(
                                    list(output.token_ids)):
                                if delta_message and delta_message.content:
                                    # This need to be added to next `delta_text`
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""
                        else:
                            # Just to add remaining `content`
623
                            if self.reasoning_parser:
624
625
626
                                delta_text = previous_text + delta_text
                                current_text = ""

627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
                            if function_name_returned[i]:
                                delta_tool_call = DeltaToolCall(
                                    function=DeltaFunctionCall(
                                        arguments=delta_text),
                                    index=i)
                            else:
                                delta_tool_call = DeltaToolCall(
                                    id=random_tool_call_id(),
                                    type="function",
                                    function=DeltaFunctionCall(
                                        name=tool_choice_function_name,
                                        arguments=delta_text),
                                    index=i)
                                function_name_returned[i] = True

642
                            delta_message = DeltaMessage(tool_calls=[
643
                                delta_tool_call,
644
645
                            ])

646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
                    elif request.tool_choice == "required":
                        assert previous_texts is not None
                        previous_text = previous_texts[i]
                        current_text = previous_text + delta_text
                        fn_name_returned = function_name_returned[i]

                        delta_message, function_name_returned[i] = (
                            self.extract_tool_call_required_streaming(
                                previous_text=previous_text,
                                current_text=current_text,
                                delta_text=delta_text,
                                function_name_returned=fn_name_returned))

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text

662
663
                    # handle streaming deltas for tools with "auto" tool choice
                    # and reasoning parser
664
                    elif tool_choice_auto and self.reasoning_parser:
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
                        assert tool_parser is not None
                        assert reasoning_parser is not None
                        assert added_content_delta_arr is not None
                        assert reasoning_end_arr is not None
                        if not reasoning_end_arr[i]:
                            delta_message = (
                                reasoning_parser.
                                extract_reasoning_content_streaming(
                                    previous_text,
                                    current_text,
                                    delta_text,
                                    previous_token_ids,
                                    current_token_ids,
                                    output.token_ids,
                                ))

                            # When encountering think end id in delta_token_ids,
                            # set reasoning status to end.
                            # Remove the text and token ids related
                            # to 'reasoning_content'.
                            if reasoning_parser.is_reasoning_end(
                                    list(output.token_ids)):
                                reasoning_end_arr[i] = True
                                current_token_ids =  \
                                    reasoning_parser.extract_content_ids(
                                        list(output.token_ids))
                                if delta_message and delta_message.content:
                                    current_text = delta_message.content
                                    delta_message.content = None
                                else:
                                    current_text = ""

                        # handle tool calls only after reasoning is done,
                        else:
                            delta_token_ids = list(output.token_ids)
                            # First time to tool call,
                            # add the remaining text and token ids
                            # to delta from previous
                            if not added_content_delta_arr[i]:
                                added_content_delta_arr[i] = True
                                previous_text = ""
                                previous_token_ids = []
                                delta_text = current_text
                                delta_token_ids = current_token_ids

                            delta_message = (
                                tool_parser.extract_tool_calls_streaming(
                                    previous_text=previous_text,
                                    current_text=current_text,
                                    delta_text=delta_text,
                                    previous_token_ids=previous_token_ids,
                                    current_token_ids=current_token_ids,
                                    delta_token_ids=delta_token_ids,
                                    request=request))
                    # when only tool calls
                    elif tool_choice_auto:
                        assert tool_parser is not None
722
723
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
724
725
                                previous_text=previous_text,
                                current_text=current_text,
726
                                delta_text=delta_text,
727
728
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
729
730
                                delta_token_ids=output.token_ids,
                                request=request))
731
                    # when only reasoning
732
                    elif self.reasoning_parser:
733
734
735
736
737
738
739
740
741
                        delta_message = (reasoning_parser.
                                         extract_reasoning_content_streaming(
                                             previous_text,
                                             current_text,
                                             delta_text,
                                             previous_token_ids,
                                             current_token_ids,
                                             output.token_ids,
                                         ))
742
                    # handle streaming just a content delta
743
744
745
                    else:
                        delta_message = DeltaMessage(content=delta_text)

746
                    # update the previous values for the next iteration
747
                    if tool_choice_auto or self.reasoning_parser:
748
749
750
751
752
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids

753
                    # set the previous values for the next iteration
754
                    previous_num_tokens[i] += len(output.token_ids)
755
756
757
758
759
760
761
762

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

763
764
765
766
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
767
                            delta=delta_message,
768
769
                            logprobs=logprobs,
                            finish_reason=None)
770
771

                    # if the model is finished generating
772
                    else:
773
774
775
776
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
777
                        auto_tools_called = False
778
                        if tool_parser:
779
780
781
782
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
783
784
785
786
787
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
788
789
790
791
792
793
794
795
796
797
                            latest_delta_len = 0
                            if ((isinstance(
                                    delta_message.tool_calls[0].function,
                                    DeltaFunctionCall)) and isinstance(
                                        delta_message.tool_calls[0].function.
                                        arguments, str)):
                                latest_delta_len = len(
                                    delta_message.tool_calls[0].function.
                                    arguments)

798
799
800
801
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
802
803
                                    "arguments", {}),
                                ensure_ascii=False)
804

805
                            # get what we've streamed so far for arguments
806
807
808
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]
809
810
                            if (latest_delta_len > 0):
                                actual_call = actual_call[:-latest_delta_len]
811
812
813
814
815
816
817
818
819
820
821
822

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)
                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

823
824
825
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
826
                            delta=delta_message,
827
                            logprobs=logprobs,
828
                            finish_reason=output.finish_reason
829
                            if not auto_tools_called else "tool_calls",
830
                            stop_reason=output.stop_reason)
831

832
                        finish_reason_sent[i] = True
833

834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

850
                    data = chunk.model_dump_json(exclude_none=True)
851
852
                    yield f"data: {data}\n\n"

853
854
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
855
856
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
857
858
859
860
861
862
863
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
864
865
866
867
868
869
870
871
872
873
874

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
875

876
877
878
879
880
881
882
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
                total_tokens=num_prompt_tokens + num_completion_tokens)

883
        except Exception as e:
884
            # TODO: Use a vllm-specific Validation Error
885
            logger.exception("Error in chat completion stream generator.")
886
887
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
888
889
890
891
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
892
893
894
895
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
896
        model_name: str,
897
        conversation: list[ConversationMessage],
898
        tokenizer: AnyTokenizer,
899
        request_metadata: RequestResponseMetadata,
900
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
901

902
        created_time = int(time.time())
903
        final_res: Optional[RequestOutput] = None
904

905
906
907
908
909
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
910
911
912
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
913

914
915
        assert final_res is not None

916
        choices: list[ChatCompletionResponseChoice] = []
917

918
919
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
920
            token_ids = output.token_ids
921
            out_logprobs = output.logprobs
922

923
924
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
925
                logprobs = self._create_chat_logprobs(
926
                    token_ids=token_ids,
927
                    top_logprobs=out_logprobs,
928
                    num_output_top_logprobs=request.top_logprobs,
929
                    tokenizer=tokenizer,
930
                    return_as_token_id=request.return_tokens_as_token_ids,
931
932
933
                )
            else:
                logprobs = None
934
            auto_tools_called = False
935

936
            if self.reasoning_parser:
937
938
939
940
941
                try:
                    reasoning_parser = self.reasoning_parser(tokenizer)
                except RuntimeError as e:
                    logger.exception("Error in reasoning parser creation.")
                    return self.create_error_response(str(e))
942
943
                # If the reasoning parser is enabled,
                # tool calls are extracted exclusively from the content.
944
945
946
                reasoning_content, content = (
                    reasoning_parser.extract_reasoning_content(
                        output.text, request=request))
947
948
949
            else:
                reasoning_content = None
                content = output.text
950

951
952
            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
953
954
955
956
            if (not self.enable_auto_tools or not self.tool_parser) and \
                (not isinstance(request.tool_choice,
                                ChatCompletionNamedToolChoiceParam
                                ) and request.tool_choice != "required"):
957
958
959
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
960
961
962

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
963
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
964

965
966
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall
967
968
                message = ChatMessage(
                    role=role,
969
                    reasoning_content=reasoning_content,
970
971
                    content="",
                    tool_calls=[
972
                        tool_call_class(function=FunctionCall(
973
                            name=request.tool_choice.function.name,
974
                            arguments=content))
975
                    ])
976

977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
            elif request.tool_choice and request.tool_choice == "required":
                tool_call_class = MistralToolCall if isinstance(
                    tokenizer, MistralTokenizer) else ToolCall

                # the fields of FunctionDefinition are a superset of the
                # tool call outputs and can be used for parsing
                tool_calls = TypeAdapter(
                    list[FunctionDefinition]).validate_json(output.text)
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
                        tool_call_class(function=FunctionCall(
                            name=tool_call.name,
                            arguments=json.dumps(tool_call.parameters)))
                        for tool_call in tool_calls
                    ])

995
996
            # if the request doesn't use tool choice
            # OR specifies to not use a tool
997
            elif not request.tool_choice or request.tool_choice == "none":
998

999
1000
1001
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1002
1003
1004
1005
1006
1007
1008

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

1009
1010
1011
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
1012
                    logger.exception("Error in tool parser creation.")
1013
1014
                    return self.create_error_response(str(e))

1015
                tool_call_info = tool_parser.extract_tool_calls(
1016
                    content if content is not None else "", request=request)
1017
1018
1019
1020
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
1021
1022
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
1023
                                          reasoning_content=reasoning_content,
1024
1025
1026
1027
1028
1029
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
1030
1031
1032
                    message = ChatMessage(role=role,
                                          reasoning_content=reasoning_content,
                                          content=content)
1033
1034
1035
1036
1037
1038
1039

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
1040
1041
1042
                message = ChatMessage(role=role,
                                      reasoning_content=reasoning_content,
                                      content=content)
1043

1044
1045
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
1046
                message=message,
1047
                logprobs=logprobs,
1048
                finish_reason="tool_calls" if auto_tools_called else
1049
                output.finish_reason if output.finish_reason else "stop",
1050
                stop_reason=output.stop_reason)
1051
1052
            choices.append(choice_data)

1053
        if request.echo:
1054
            last_msg_content: Union[str, list[dict[str, str]]] = ""
1055
1056
            if conversation and "content" in conversation[-1] and conversation[
                    -1].get("role") == role:
1057
                last_msg_content = conversation[-1]["content"] or ""
1058
1059
1060
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
1061
1062

            for choice in choices:
1063
1064
                full_message = last_msg_content + (choice.message.content
                                                   or "")
1065
1066
                choice.message.content = full_message

1067
        assert final_res.prompt_token_ids is not None
1068
        num_prompt_tokens = len(final_res.prompt_token_ids)
1069
1070
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
1071
1072
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
1073
1074
1075
1076
1077
1078
1079
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
1080
1081
1082

        request_metadata.final_usage_info = usage

1083
1084
1085
1086
1087
1088
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
1089
            prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
Robert Shaw's avatar
Robert Shaw committed
1090
            kv_transfer_params=final_res.kv_transfer_params,
1091
1092
        )

1093
        return response
1094
1095

    def _get_top_logprobs(
1096
            self, logprobs: dict[int, Logprob], top_logprobs: Optional[int],
1097
1098
            tokenizer: AnyTokenizer,
            should_return_as_token_id: bool) -> list[ChatCompletionLogProb]:
1099
        return [
1100
1101
1102
1103
            ChatCompletionLogProb(token=(token := self._get_decoded_token(
                p[1],
                p[0],
                tokenizer,
1104
                return_as_token_id=should_return_as_token_id)),
1105
1106
1107
                                  logprob=max(p[1].logprob, -9999.0),
                                  bytes=list(
                                      token.encode("utf-8", errors="replace")))
1108
1109
1110
1111
1112
1113
1114
            for i, p in enumerate(logprobs.items())
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
1115
        top_logprobs: GenericSequence[Optional[dict[int, Logprob]]],
1116
        tokenizer: AnyTokenizer,
1117
        num_output_top_logprobs: Optional[int] = None,
1118
        return_as_token_id: Optional[bool] = None,
1119
1120
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
1121
        logprobs_content: list[ChatCompletionLogProbsContent] = []
1122

1123
1124
        should_return_as_token_id = return_as_token_id if \
            return_as_token_id is not None else self.return_tokens_as_token_ids
1125
1126
        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
1127
1128
            if step_top_logprobs is None or step_top_logprobs.get(
                    token_id) is None:
1129
                token = tokenizer.decode(token_id)
1130
                if should_return_as_token_id:
1131
                    token = f"token_id:{token_id}"
1132

1133
1134
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1135
                        token=token,
1136
1137
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
1138
            else:
1139
1140
1141
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

1142
1143
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
1144
                        token=self._get_decoded_token(
1145
1146
1147
                            step_token,
                            token_id,
                            tokenizer,
1148
                            should_return_as_token_id,
1149
1150
1151
1152
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
1153
                        top_logprobs=self._get_top_logprobs(
1154
1155
                            step_top_logprobs, num_output_top_logprobs,
                            tokenizer, should_return_as_token_id),
1156
                    ))
1157
1158

        return ChatCompletionLogProbs(content=logprobs_content)
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
1188
1189
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
1190
1191
1192
1193
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )