serving_chat.py 38.2 KB
Newer Older
1
import asyncio
2
import json
3
import time
4
5
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, Final, List,
                    Optional)
6
from typing import Sequence as GenericSequence
7
from typing import Union
8

9
from fastapi import Request
10

11
from vllm.config import ModelConfig
12
from vllm.engine.protocol import EngineClient
13
14
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
                                         ConversationMessage)
15
from vllm.entrypoints.logger import RequestLogger
16
from vllm.entrypoints.openai.protocol import (
17
18
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
19
    ChatCompletionRequest, ChatCompletionResponse,
20
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
21
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
22
23
    DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo,
    RequestResponseMetadata, ToolCall, UsageInfo)
24
25
from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
                                                    LoRAModulePath,
26
                                                    OpenAIServing,
27
                                                    PromptAdapterPath)
28
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
29
from vllm.logger import init_logger
30
from vllm.outputs import CompletionOutput, RequestOutput
31
from vllm.sampling_params import BeamSearchParams, SamplingParams
32
from vllm.sequence import Logprob
33
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
34
from vllm.transformers_utils.tokenizers import maybe_serialize_tool_calls
35
from vllm.utils import iterate_with_cancellation
36
37
38
39
40
41

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
        base_model_paths: List[BaseModelPath],
        response_role: str,
        *,
        lora_modules: Optional[List[LoRAModulePath]],
        prompt_adapters: Optional[List[PromptAdapterPath]],
        request_logger: Optional[RequestLogger],
        chat_template: Optional[str],
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
        enable_auto_tools: bool = False,
        tool_parser: Optional[str] = None,
        enable_prompt_tokens_details: bool = False,
    ) -> None:
59
        super().__init__(engine_client=engine_client,
60
                         model_config=model_config,
61
                         base_model_paths=base_model_paths,
62
63
                         lora_modules=lora_modules,
                         prompt_adapters=prompt_adapters,
64
65
                         request_logger=request_logger,
                         return_tokens_as_token_ids=return_tokens_as_token_ids)
66

67
        self.response_role = response_role
68
69
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
70

71
72
73
74
75
76
77
78
79
80
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
81
            try:
82
83
84
85
86
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
87
88
89
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
90
                raise TypeError("Error: --enable-auto-tool-choice requires "
91
92
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
93

94
95
        self.enable_prompt_tokens_details = enable_prompt_tokens_details

96
    async def create_chat_completion(
97
98
        self,
        request: ChatCompletionRequest,
99
100
101
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
102
103
        """
        Chat Completion API similar to OpenAI's API.
104

105
106
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
107
        Chat Completion API.
108
109
110
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
111
            logger.error("Error with model %s", error_check_ret)
112
113
            return error_check_ret

114
115
116
117
118
119
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

120
        try:
121
122
123
124
125
            (
                lora_request,
                prompt_adapter_request,
            ) = self._maybe_get_adapters(request)

126
127
            model_name = self._get_model_name(lora_request)

128
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
129

130
131
132
133
134
135
136
137
            tool_parser = self.tool_parser

            # validation for OpenAI tools
            # tool_choice = "required" is not supported
            if request.tool_choice == "required":
                return self.create_error_response(
                    "tool_choice = \"required\" is not supported!")

138
139
140
            # because of issues with pydantic we need to potentially
            # re-serialize the tool_calls field of the request
            # for more info: see comment in `maybe_serialize_tool_calls`
141
            if isinstance(tokenizer, MistralTokenizer):
142
                maybe_serialize_tool_calls(request)
143

144
145
146
147
148
149
150
151
152
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
                    and not isinstance(tokenizer, MistralTokenizer)):
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
153

154
155
156
157
            tool_dicts = None if request.tools is None else [
                tool.model_dump() for tool in request.tools
            ]

158
159
160
161
162
163
164
165
166
            (
                conversation,
                request_prompts,
                engine_prompts,
            ) = await self._preprocess_chat(
                request,
                tokenizer,
                request.messages,
                chat_template=request.chat_template or self.chat_template,
167
                chat_template_content_format=self.chat_template_content_format,
168
169
170
171
172
173
174
175
176
177
178
                add_generation_prompt=request.add_generation_prompt,
                continue_final_message=request.continue_final_message,
                tool_dicts=tool_dicts,
                documents=request.documents,
                chat_template_kwargs=request.chat_template_kwargs,
                tool_parser=tool_parser,
                truncate_prompt_tokens=request.truncate_prompt_tokens,
                add_special_tokens=request.add_special_tokens,
            )
        except ValueError as e:
            logger.exception("Error in preprocessing prompt inputs")
179
180
            return self.create_error_response(str(e))

181
182
        request_id = "chatcmpl-" \
                     f"{self._base_request_id(raw_request, request.request_id)}"
183
184
185
186
187

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

188
189
        # Schedule the request and get the result generator.
        generators: List[AsyncGenerator[RequestOutput, None]] = []
190
        try:
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
                default_max_tokens = self.max_model_len - len(
                    engine_prompt["prompt_token_ids"])
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
                        default_max_tokens)
                else:
                    sampling_params = request.to_sampling_params(
                        default_max_tokens)

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
                                 lora_request=lora_request,
                                 prompt_adapter_request=prompt_adapter_request)

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        prompt_adapter_request=prompt_adapter_request,
                        priority=request.priority,
                    )

                generators.append(generator)
229
        except ValueError as e:
230
            # TODO: Use a vllm-specific Validation Error
231
232
            return self.create_error_response(str(e))

233
234
235
        assert len(generators) == 1
        result_generator, = generators

236
237
238
239
        if raw_request:
            result_generator = iterate_with_cancellation(
                result_generator, raw_request.is_disconnected)

240
241
242
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
243
244
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
245

246
247
        try:
            return await self.chat_completion_full_generator(
248
249
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
250
251
252
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
253
254
255
256

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
257
        return request.messages[-1]["role"]
258
259

    async def chat_completion_stream_generator(
260
261
262
263
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
264
        model_name: str,
265
        conversation: List[ConversationMessage],
266
        tokenizer: AnyTokenizer,
267
        request_metadata: RequestResponseMetadata,
268
    ) -> AsyncGenerator[str, None]:
269
        created_time = int(time.time())
270
        chunk_object_type: Final = "chat.completion.chunk"
271
        first_iteration = True
272
273

        # Send response for each token for each request.n (index)
274
275
276
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
277
        num_prompt_tokens = 0
278
        num_cached_tokens = None
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

        all_previous_token_ids: Optional[List[List[int]]]
        if tool_choice_auto:
            # These are only required in "auto" tool choice case
            previous_texts = [""] * num_choices
            all_previous_token_ids = [[]] * num_choices
        else:
            previous_texts, all_previous_token_ids = None, None

298
299
300
301
302
303
304
305
306
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
                tool_parsers: List[Optional[ToolParser]] = [
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
        except RuntimeError as e:
307
            logger.exception("Error in tool parser creation.")
308
309
310
311
312
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

313
314
315
316
317
318
319
320
        stream_options = request.stream_options
        if stream_options:
            include_usage = stream_options.include_usage
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

321
322
        try:
            async for res in result_generator:
323
324
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
325
326
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
327

328
329
330
331
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
332
                    num_cached_tokens = res.num_cached_tokens
333
334
                    # Send first response for each request.n (index) with
                    # the role
335
                    role = self.get_chat_request_role(request)
336
337
338

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
339
                    for i in range(num_choices):
340
341
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
342
343
344
345
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
346
347
348
349
350
351
352
353
                            logprobs=None,
                            finish_reason=None)
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
                            model=model_name)
354

355
356
357
358
359
360
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
361

362
363
364
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

365
366
                    # Send response to echo the input portion of the
                    # last message
367
                    if request.echo:
368
                        last_msg_content: Union[str, List[Dict[str, str]]] = ""
369
370
371
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
372
373

                        if last_msg_content:
374
                            for i in range(num_choices):
375
376
377
378
379
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
380
                                        logprobs=None,
381
                                        finish_reason=None))
382
383
384
385
386
387
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
388
389
390
391
392
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
393

394
395
396
397
398
399
400
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
401
                    tool_parser = tool_parsers[i]
402
403
404
405

                    if finish_reason_sent[i]:
                        continue

406
                    if request.logprobs and request.top_logprobs is not None:
407
                        assert output.logprobs is not None, (
408
                            "Did not output logprobs")
409
                        logprobs = self._create_chat_logprobs(
410
411
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
412
                            tokenizer=tokenizer,
413
                            num_output_top_logprobs=request.top_logprobs,
414
415
416
417
                        )
                    else:
                        logprobs = None

418
                    delta_text = output.text
419
420
421
422
423
424

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

425
                    delta_message: Optional[DeltaMessage]
426

427
                    # handle streaming deltas for tools with named tool_choice
428
                    if tool_choice_function_name:
429
                        delta_message = DeltaMessage(tool_calls=[
430
                            DeltaToolCall(function=DeltaFunctionCall(
431
                                name=tool_choice_function_name,
432
433
                                arguments=delta_text),
                                          index=i)
434
                        ])
435
436

                    # handle streaming deltas for tools with "auto" tool choice
437
438
439
440
441
442
443
444
445
446
447
                    elif tool_choice_auto:
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        assert tool_parser is not None
                        #TODO optimize manipulation of these lists
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
                        current_token_ids = previous_token_ids + list(
                            output.token_ids)

448
449
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
450
451
                                previous_text=previous_text,
                                current_text=current_text,
452
                                delta_text=delta_text,
453
454
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
455
456
                                delta_token_ids=output.token_ids,
                                request=request))
457
458
459
460

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
461
462

                    # handle streaming just a content delta
463
464
465
                    else:
                        delta_message = DeltaMessage(content=delta_text)

466
                    # set the previous values for the next iteration
467
                    previous_num_tokens[i] += len(output.token_ids)
468
469
470
471
472
473
474
475

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

476
477
478
479
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
480
                            delta=delta_message,
481
482
                            logprobs=logprobs,
                            finish_reason=None)
483
484

                    # if the model is finished generating
485
                    else:
486
487
488
489
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
490
                        auto_tools_called = False
491
                        if tool_parser:
492
493
494
495
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
496
497
498
499
500
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
501
502
503
504
505
506
507
508
509
510
                            latest_delta_len = 0
                            if ((isinstance(
                                    delta_message.tool_calls[0].function,
                                    DeltaFunctionCall)) and isinstance(
                                        delta_message.tool_calls[0].function.
                                        arguments, str)):
                                latest_delta_len = len(
                                    delta_message.tool_calls[0].function.
                                    arguments)

511
512
513
514
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
515
516
                                    "arguments", {}),
                                ensure_ascii=False)
517

518
                            # get what we've streamed so far for arguments
519
520
521
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]
522
523
                            if (latest_delta_len > 0):
                                actual_call = actual_call[:-latest_delta_len]
524
525
526
527
528
529
530
531
532
533
534
535

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)
                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

536
537
538
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
539
                            delta=delta_message,
540
                            logprobs=logprobs,
541
                            finish_reason=output.finish_reason
542
                            if not auto_tools_called else "tool_calls",
543
                            stop_reason=output.stop_reason)
544

545
                        finish_reason_sent[i] = True
546

547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

                    data = chunk.model_dump_json(exclude_unset=True)
                    yield f"data: {data}\n\n"

566
567
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
568
569
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
570
571
572
573
574
575
576
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
577
578
579
580
581
582
583
584
585
586
587

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
588

589
590
591
592
593
594
595
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
                total_tokens=num_prompt_tokens + num_completion_tokens)

596
597
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
598
            logger.exception("Error in chat completion stream generator.")
599
600
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
601
602
603
604
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
605
606
607
608
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
609
        model_name: str,
610
        conversation: List[ConversationMessage],
611
        tokenizer: AnyTokenizer,
612
        request_metadata: RequestResponseMetadata,
613
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
614

615
        created_time = int(time.time())
616
        final_res: Optional[RequestOutput] = None
617

618
619
620
621
622
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
623
624
625
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
626

627
628
        assert final_res is not None

629
        choices: List[ChatCompletionResponseChoice] = []
630

631
632
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
633
            token_ids = output.token_ids
634
            out_logprobs = output.logprobs
635

636
637
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
638
                logprobs = self._create_chat_logprobs(
639
                    token_ids=token_ids,
640
                    top_logprobs=out_logprobs,
641
                    num_output_top_logprobs=request.top_logprobs,
642
                    tokenizer=tokenizer,
643
644
645
646
                )
            else:
                logprobs = None

647
648
649
650
            # In the OpenAI API the finish_reason is "tools_called"
            # if the tool choice is auto and the model produced a tool
            # call. The same is not true for named function calls
            auto_tools_called = False
651
652
653

            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
654
            if (not self.enable_auto_tools
655
656
657
658
659
660
661
                    or not self.tool_parser) and not isinstance(
                        request.tool_choice,
                        ChatCompletionNamedToolChoiceParam):
                message = ChatMessage(role=role, content=output.text)

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
662
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
663

664
665
666
667
668
669
670
671
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
                        ToolCall(function=FunctionCall(
                            name=request.tool_choice.function.name,
                            arguments=output.text))
                    ])
672
673
674

            # if the request doesn't use tool choice
            # OR specifies to not use a tool
675
            elif not request.tool_choice or request.tool_choice == "none":
676
677
678
679
680
681
682
683
684

                message = ChatMessage(role=role, content=output.text)

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

685
686
687
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
688
                    logger.exception("Error in tool parser creation.")
689
690
                    return self.create_error_response(str(e))

691
692
                tool_call_info = tool_parser.extract_tool_calls(
                    output.text, request=request)
693
694
695
696
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
                    message = ChatMessage(role=role, content=output.text)

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
713
714
                message = ChatMessage(role=role, content=output.text)

715
716
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
717
                message=message,
718
                logprobs=logprobs,
719
                finish_reason="tool_calls" if auto_tools_called else
720
                output.finish_reason if output.finish_reason else "stop",
721
                stop_reason=output.stop_reason)
722
723
            choices.append(choice_data)

724
        if request.echo:
725
            last_msg_content: Union[str, List[Dict[str, str]]] = ""
726
727
            if conversation and "content" in conversation[-1] and conversation[
                    -1].get("role") == role:
728
                last_msg_content = conversation[-1]["content"] or ""
729
730
731
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
732
733

            for choice in choices:
734
735
                full_message = last_msg_content + (choice.message.content
                                                   or "")
736
737
                choice.message.content = full_message

738
        assert final_res.prompt_token_ids is not None
739
        num_prompt_tokens = len(final_res.prompt_token_ids)
740
741
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
742
743
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
744
745
746
747
748
749
750
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
751
752
753

        request_metadata.final_usage_info = usage

754
755
756
757
758
759
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
760
            prompt_logprobs=final_res.prompt_logprobs,
761
762
        )

763
        return response
764
765

    def _get_top_logprobs(
766
            self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
767
            tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]:
768
        return [
769
770
771
772
773
774
775
776
            ChatCompletionLogProb(token=(token := self._get_decoded_token(
                p[1],
                p[0],
                tokenizer,
                return_as_token_id=self.return_tokens_as_token_ids)),
                                  logprob=max(p[1].logprob, -9999.0),
                                  bytes=list(
                                      token.encode("utf-8", errors="replace")))
777
778
779
780
781
782
783
784
            for i, p in enumerate(logprobs.items())
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
        top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
785
        tokenizer: AnyTokenizer,
786
787
788
        num_output_top_logprobs: Optional[int] = None,
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
789
        logprobs_content: List[ChatCompletionLogProbsContent] = []
790
791
792
793

        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
            if step_top_logprobs is None:
794
                token = tokenizer.decode(token_id)
795
796
                if self.return_tokens_as_token_ids:
                    token = f"token_id:{token_id}"
797

798
799
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
800
                        token=token,
801
802
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
803
            else:
804
805
806
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

807
808
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
809
                        token=self._get_decoded_token(
810
811
812
813
814
815
816
817
                            step_token,
                            token_id,
                            tokenizer,
                            self.return_tokens_as_token_ids,
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
818
                        top_logprobs=self._get_top_logprobs(
819
820
821
822
823
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                        ),
                    ))
824
825

        return ChatCompletionLogProbs(content=logprobs_content)
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
855
856
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
857
858
859
860
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )