serving_chat.py 37 KB
Newer Older
1
import asyncio
2
import json
3
import time
4
5
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, Final, List,
                    Optional)
6
from typing import Sequence as GenericSequence
7
from typing import Union
8

9
from fastapi import Request
10

11
from vllm.config import ModelConfig
12
from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.chat_utils import ConversationMessage, load_chat_template
14
from vllm.entrypoints.logger import RequestLogger
15
from vllm.entrypoints.openai.protocol import (
16
17
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
18
    ChatCompletionRequest, ChatCompletionResponse,
19
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
20
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
21
22
    DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo,
    RequestResponseMetadata, ToolCall, UsageInfo)
23
24
from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
                                                    LoRAModulePath,
25
                                                    OpenAIServing,
26
                                                    PromptAdapterPath)
27
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
28
from vllm.logger import init_logger
29
from vllm.outputs import CompletionOutput, RequestOutput
30
from vllm.sampling_params import BeamSearchParams, SamplingParams
31
from vllm.sequence import Logprob
32
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
33
from vllm.utils import iterate_with_cancellation
34
35
36
37
38
39

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

40
    def __init__(self,
41
                 engine_client: EngineClient,
42
                 model_config: ModelConfig,
43
                 base_model_paths: List[BaseModelPath],
44
45
46
47
48
49
50
51
                 response_role: str,
                 *,
                 lora_modules: Optional[List[LoRAModulePath]],
                 prompt_adapters: Optional[List[PromptAdapterPath]],
                 request_logger: Optional[RequestLogger],
                 chat_template: Optional[str],
                 return_tokens_as_token_ids: bool = False,
                 enable_auto_tools: bool = False,
52
53
                 tool_parser: Optional[str] = None,
                 enable_prompt_tokens_details: bool = False):
54
        super().__init__(engine_client=engine_client,
55
                         model_config=model_config,
56
                         base_model_paths=base_model_paths,
57
58
                         lora_modules=lora_modules,
                         prompt_adapters=prompt_adapters,
59
60
                         request_logger=request_logger,
                         return_tokens_as_token_ids=return_tokens_as_token_ids)
61

62
        self.response_role = response_role
63
        self.use_tool_use_model_template = False
64
        self.chat_template = load_chat_template(chat_template)
65

66
67
68
69
70
71
72
73
74
75
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
76
            try:
77
78
79
80
81
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
82
83
84
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
85
                raise TypeError("Error: --enable-auto-tool-choice requires "
86
87
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
88

89
90
        self.enable_prompt_tokens_details = enable_prompt_tokens_details

91
    async def create_chat_completion(
92
93
        self,
        request: ChatCompletionRequest,
94
95
96
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
97
98
        """
        Chat Completion API similar to OpenAI's API.
99

100
101
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
102
        Chat Completion API.
103
104
105
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
106
            logger.error("Error with model %s", error_check_ret)
107
108
            return error_check_ret

109
110
111
112
113
114
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

115
        try:
116
117
118
119
120
            (
                lora_request,
                prompt_adapter_request,
            ) = self._maybe_get_adapters(request)

121
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
            tool_parser = self.tool_parser

            # validation for OpenAI tools
            # tool_choice = "required" is not supported
            if request.tool_choice == "required":
                return self.create_error_response(
                    "tool_choice = \"required\" is not supported!")

            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
                    and not isinstance(tokenizer, MistralTokenizer)):
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
139

140
141
142
143
            tool_dicts = None if request.tools is None else [
                tool.model_dump() for tool in request.tools
            ]

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
            (
                conversation,
                request_prompts,
                engine_prompts,
            ) = await self._preprocess_chat(
                request,
                tokenizer,
                request.messages,
                chat_template=request.chat_template or self.chat_template,
                add_generation_prompt=request.add_generation_prompt,
                continue_final_message=request.continue_final_message,
                tool_dicts=tool_dicts,
                documents=request.documents,
                chat_template_kwargs=request.chat_template_kwargs,
                tool_parser=tool_parser,
                truncate_prompt_tokens=request.truncate_prompt_tokens,
                add_special_tokens=request.add_special_tokens,
            )
        except ValueError as e:
            logger.exception("Error in preprocessing prompt inputs")
164
165
            return self.create_error_response(str(e))

166
        request_id = f"chatcmpl-{request.request_id}"
167
168
169
170
171

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

172
173
        # Schedule the request and get the result generator.
        generators: List[AsyncGenerator[RequestOutput, None]] = []
174
        try:
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
                default_max_tokens = self.max_model_len - len(
                    engine_prompt["prompt_token_ids"])
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
                        default_max_tokens)
                else:
                    sampling_params = request.to_sampling_params(
                        default_max_tokens)

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
                                 lora_request=lora_request,
                                 prompt_adapter_request=prompt_adapter_request)

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        prompt_adapter_request=prompt_adapter_request,
                        priority=request.priority,
                    )

                generators.append(generator)
213
        except ValueError as e:
214
            # TODO: Use a vllm-specific Validation Error
215
216
            return self.create_error_response(str(e))

217
218
219
        assert len(generators) == 1
        result_generator, = generators

220
221
222
223
        if raw_request:
            result_generator = iterate_with_cancellation(
                result_generator, raw_request.is_disconnected)

224
225
226
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
227
228
                request, result_generator, request_id, conversation, tokenizer,
                request_metadata)
229

230
231
        try:
            return await self.chat_completion_full_generator(
232
233
                request, result_generator, request_id, conversation, tokenizer,
                request_metadata)
234
235
236
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
237
238
239
240

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
241
        return request.messages[-1]["role"]
242
243

    async def chat_completion_stream_generator(
244
245
246
247
248
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
        conversation: List[ConversationMessage],
249
        tokenizer: AnyTokenizer,
250
        request_metadata: RequestResponseMetadata,
251
    ) -> AsyncGenerator[str, None]:
252
        model_name = self.base_model_paths[0].name
253
        created_time = int(time.time())
254
        chunk_object_type: Final = "chat.completion.chunk"
255
        first_iteration = True
256
257

        # Send response for each token for each request.n (index)
258
259
260
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
261
        num_prompt_tokens = 0
262
        num_cached_tokens = None
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

        all_previous_token_ids: Optional[List[List[int]]]
        if tool_choice_auto:
            # These are only required in "auto" tool choice case
            previous_texts = [""] * num_choices
            all_previous_token_ids = [[]] * num_choices
        else:
            previous_texts, all_previous_token_ids = None, None

282
283
284
285
286
287
288
289
290
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
                tool_parsers: List[Optional[ToolParser]] = [
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
        except RuntimeError as e:
291
            logger.exception("Error in tool parser creation.")
292
293
294
295
296
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

297
298
299
300
301
302
303
304
        stream_options = request.stream_options
        if stream_options:
            include_usage = stream_options.include_usage
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

305
306
        try:
            async for res in result_generator:
307
308
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
309
310
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
311

312
313
314
315
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
316
                    num_cached_tokens = res.num_cached_tokens
317
318
                    # Send first response for each request.n (index) with
                    # the role
319
                    role = self.get_chat_request_role(request)
320
321
322

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
323
                    for i in range(num_choices):
324
325
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
326
327
328
329
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
330
331
332
333
334
335
336
337
                            logprobs=None,
                            finish_reason=None)
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
                            model=model_name)
338

339
340
341
342
343
344
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
345

346
347
348
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

349
350
                    # Send response to echo the input portion of the
                    # last message
351
                    if request.echo or request.continue_final_message:
352
                        last_msg_content: Union[str, List[Dict[str, str]]] = ""
353
354
355
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
356
357

                        if last_msg_content:
358
                            for i in range(num_choices):
359
360
361
362
363
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
364
                                        logprobs=None,
365
                                        finish_reason=None))
366
367
368
369
370
371
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
372
373
374
375
376
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
377

378
379
380
381
382
383
384
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
385
                    tool_parser = tool_parsers[i]
386
387
388
389

                    if finish_reason_sent[i]:
                        continue

390
                    if request.logprobs and request.top_logprobs is not None:
391
                        assert output.logprobs is not None, (
392
                            "Did not output logprobs")
393
                        logprobs = self._create_chat_logprobs(
394
395
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
396
                            tokenizer=tokenizer,
397
                            num_output_top_logprobs=request.top_logprobs,
398
399
400
401
                        )
                    else:
                        logprobs = None

402
                    delta_text = output.text
403
404
405
406
407
408

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

409
                    delta_message: Optional[DeltaMessage]
410

411
                    # handle streaming deltas for tools with named tool_choice
412
                    if tool_choice_function_name:
413
                        delta_message = DeltaMessage(tool_calls=[
414
                            DeltaToolCall(function=DeltaFunctionCall(
415
                                name=tool_choice_function_name,
416
417
                                arguments=delta_text),
                                          index=i)
418
                        ])
419
420

                    # handle streaming deltas for tools with "auto" tool choice
421
422
423
424
425
426
427
428
429
430
431
                    elif tool_choice_auto:
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        assert tool_parser is not None
                        #TODO optimize manipulation of these lists
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
                        current_token_ids = previous_token_ids + list(
                            output.token_ids)

432
433
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
434
435
                                previous_text=previous_text,
                                current_text=current_text,
436
                                delta_text=delta_text,
437
438
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
439
440
                                delta_token_ids=output.token_ids,
                                request=request))
441
442
443
444

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
445
446

                    # handle streaming just a content delta
447
448
449
                    else:
                        delta_message = DeltaMessage(content=delta_text)

450
                    # set the previous values for the next iteration
451
                    previous_num_tokens[i] += len(output.token_ids)
452
453
454
455
456
457
458
459

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

460
461
462
463
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
464
                            delta=delta_message,
465
466
                            logprobs=logprobs,
                            finish_reason=None)
467
468

                    # if the model is finished generating
469
                    else:
470
471
472
473
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
474
                        auto_tools_called = False
475
                        if tool_parser:
476
477
478
479
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
480
481
482
483
484
485
486
487
488
489
490
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
                                    "arguments", {}))

491
                            # get what we've streamed so far for arguments
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)

                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

508
509
510
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
511
                            delta=delta_message,
512
                            logprobs=logprobs,
513
                            finish_reason=output.finish_reason
514
                            if not auto_tools_called else "tool_calls",
515
                            stop_reason=output.stop_reason)
516

517
                        finish_reason_sent[i] = True
518

519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

                    data = chunk.model_dump_json(exclude_unset=True)
                    yield f"data: {data}\n\n"

538
539
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
540
541
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
542
543
544
545
546
547
548
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
549
550
551
552
553
554
555
556
557
558
559

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
560

561
562
563
564
565
566
567
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
                total_tokens=num_prompt_tokens + num_completion_tokens)

568
569
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
570
            logger.exception("Error in chat completion stream generator.")
571
572
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
573
574
575
576
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
577
578
579
580
581
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
        conversation: List[ConversationMessage],
582
        tokenizer: AnyTokenizer,
583
        request_metadata: RequestResponseMetadata,
584
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
585

586
        model_name = self.base_model_paths[0].name
587
        created_time = int(time.time())
588
        final_res: Optional[RequestOutput] = None
589

590
591
592
593
594
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
595
596
597
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
598

599
600
        assert final_res is not None

601
        choices: List[ChatCompletionResponseChoice] = []
602

603
604
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
605
            token_ids = output.token_ids
606
            out_logprobs = output.logprobs
607

608
609
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
610
                logprobs = self._create_chat_logprobs(
611
                    token_ids=token_ids,
612
                    top_logprobs=out_logprobs,
613
                    num_output_top_logprobs=request.top_logprobs,
614
                    tokenizer=tokenizer,
615
616
617
618
                )
            else:
                logprobs = None

619
620
621
622
            # In the OpenAI API the finish_reason is "tools_called"
            # if the tool choice is auto and the model produced a tool
            # call. The same is not true for named function calls
            auto_tools_called = False
623
624
625

            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
626
            if (not self.enable_auto_tools
627
628
629
630
631
632
633
                    or not self.tool_parser) and not isinstance(
                        request.tool_choice,
                        ChatCompletionNamedToolChoiceParam):
                message = ChatMessage(role=role, content=output.text)

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
634
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
635

636
637
638
639
640
641
642
643
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
                        ToolCall(function=FunctionCall(
                            name=request.tool_choice.function.name,
                            arguments=output.text))
                    ])
644
645
646

            # if the request doesn't use tool choice
            # OR specifies to not use a tool
647
            elif not request.tool_choice or request.tool_choice == "none":
648
649
650
651
652
653
654
655
656

                message = ChatMessage(role=role, content=output.text)

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

657
658
659
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
660
                    logger.exception("Error in tool parser creation.")
661
662
                    return self.create_error_response(str(e))

663
664
                tool_call_info = tool_parser.extract_tool_calls(
                    output.text, request=request)
665
666
667
668
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
                    message = ChatMessage(role=role, content=output.text)

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
685
686
                message = ChatMessage(role=role, content=output.text)

687
688
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
689
                message=message,
690
                logprobs=logprobs,
691
                finish_reason="tool_calls" if auto_tools_called else
692
                output.finish_reason if output.finish_reason else "stop",
693
                stop_reason=output.stop_reason)
694
695
            choices.append(choice_data)

696
        if request.echo or request.continue_final_message:
697
            last_msg_content: Union[str, List[Dict[str, str]]] = ""
698
699
            if conversation and "content" in conversation[-1] and conversation[
                    -1].get("role") == role:
700
                last_msg_content = conversation[-1]["content"] or ""
701
702
703
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
704
705

            for choice in choices:
706
707
                full_message = last_msg_content + (choice.message.content
                                                   or "")
708
709
                choice.message.content = full_message

710
        assert final_res.prompt_token_ids is not None
711
        num_prompt_tokens = len(final_res.prompt_token_ids)
712
713
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
714
715
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
716
717
718
719
720
721
722
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
723
724
725

        request_metadata.final_usage_info = usage

726
727
728
729
730
731
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
732
            prompt_logprobs=final_res.prompt_logprobs,
733
734
        )

735
        return response
736
737

    def _get_top_logprobs(
738
            self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
739
            tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]:
740
        return [
741
742
743
744
745
746
747
748
            ChatCompletionLogProb(token=(token := self._get_decoded_token(
                p[1],
                p[0],
                tokenizer,
                return_as_token_id=self.return_tokens_as_token_ids)),
                                  logprob=max(p[1].logprob, -9999.0),
                                  bytes=list(
                                      token.encode("utf-8", errors="replace")))
749
750
751
752
753
754
755
756
            for i, p in enumerate(logprobs.items())
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
        top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
757
        tokenizer: AnyTokenizer,
758
759
760
        num_output_top_logprobs: Optional[int] = None,
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
761
        logprobs_content: List[ChatCompletionLogProbsContent] = []
762
763
764
765

        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
            if step_top_logprobs is None:
766
                token = tokenizer.decode(token_id)
767
768
                if self.return_tokens_as_token_ids:
                    token = f"token_id:{token_id}"
769

770
771
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
772
                        token=token,
773
774
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
775
            else:
776
777
778
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

779
780
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
781
                        token=self._get_decoded_token(
782
783
784
785
786
787
788
789
                            step_token,
                            token_id,
                            tokenizer,
                            self.return_tokens_as_token_ids,
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
790
                        top_logprobs=self._get_top_logprobs(
791
792
793
794
795
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                        ),
                    ))
796
797

        return ChatCompletionLogProbs(content=logprobs_content)
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
827
828
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
829
830
831
832
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )