serving_chat.py 38.5 KB
Newer Older
1
import asyncio
2
import json
3
import time
4
5
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, Final, List,
                    Optional)
6
from typing import Sequence as GenericSequence
7
from typing import Union
8

9
from fastapi import Request
10

11
from vllm.config import ModelConfig
12
from vllm.engine.protocol import EngineClient
13
14
from vllm.entrypoints.chat_utils import (ChatTemplateContentFormatOption,
                                         ConversationMessage)
15
from vllm.entrypoints.logger import RequestLogger
16
from vllm.entrypoints.openai.protocol import (
17
18
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
19
    ChatCompletionRequest, ChatCompletionResponse,
20
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
21
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
22
23
    DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo,
    RequestResponseMetadata, ToolCall, UsageInfo)
24
25
from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
                                                    LoRAModulePath,
26
                                                    OpenAIServing,
27
                                                    PromptAdapterPath)
28
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
29
from vllm.logger import init_logger
30
from vllm.outputs import CompletionOutput, RequestOutput
31
from vllm.sampling_params import BeamSearchParams, SamplingParams
32
from vllm.sequence import Logprob
33
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
34
from vllm.transformers_utils.tokenizers import maybe_serialize_tool_calls
35
36
37
38
39
40

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
    def __init__(
        self,
        engine_client: EngineClient,
        model_config: ModelConfig,
        base_model_paths: List[BaseModelPath],
        response_role: str,
        *,
        lora_modules: Optional[List[LoRAModulePath]],
        prompt_adapters: Optional[List[PromptAdapterPath]],
        request_logger: Optional[RequestLogger],
        chat_template: Optional[str],
        chat_template_content_format: ChatTemplateContentFormatOption,
        return_tokens_as_token_ids: bool = False,
        enable_auto_tools: bool = False,
        tool_parser: Optional[str] = None,
        enable_prompt_tokens_details: bool = False,
    ) -> None:
58
        super().__init__(engine_client=engine_client,
59
                         model_config=model_config,
60
                         base_model_paths=base_model_paths,
61
62
                         lora_modules=lora_modules,
                         prompt_adapters=prompt_adapters,
63
64
                         request_logger=request_logger,
                         return_tokens_as_token_ids=return_tokens_as_token_ids)
65

66
        self.response_role = response_role
67
68
        self.chat_template = chat_template
        self.chat_template_content_format: Final = chat_template_content_format
69

70
71
72
73
74
75
76
77
78
79
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
80
            try:
81
82
83
84
85
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
86
87
88
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
89
                raise TypeError("Error: --enable-auto-tool-choice requires "
90
91
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
92

93
        self.enable_prompt_tokens_details = enable_prompt_tokens_details
94
95
96
97
        diff_sampling_param = self.model_config.get_diff_sampling_param()
        if diff_sampling_param:
            logger.info("Overwriting default chat sampling param with: %s",
                        diff_sampling_param)
98

99
    async def create_chat_completion(
100
101
        self,
        request: ChatCompletionRequest,
102
103
104
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
105
106
        """
        Chat Completion API similar to OpenAI's API.
107

108
109
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
110
        Chat Completion API.
111
112
113
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
114
            logger.error("Error with model %s", error_check_ret)
115
116
            return error_check_ret

117
118
119
120
121
122
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

123
        try:
124
125
126
127
128
            (
                lora_request,
                prompt_adapter_request,
            ) = self._maybe_get_adapters(request)

129
130
            model_name = self._get_model_name(lora_request)

131
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
132

133
134
135
136
137
138
139
140
            tool_parser = self.tool_parser

            # validation for OpenAI tools
            # tool_choice = "required" is not supported
            if request.tool_choice == "required":
                return self.create_error_response(
                    "tool_choice = \"required\" is not supported!")

141
142
143
            # because of issues with pydantic we need to potentially
            # re-serialize the tool_calls field of the request
            # for more info: see comment in `maybe_serialize_tool_calls`
144
            if isinstance(tokenizer, MistralTokenizer):
145
                maybe_serialize_tool_calls(request)
146

147
148
149
150
151
152
153
154
155
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
                    and not isinstance(tokenizer, MistralTokenizer)):
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
156

157
158
159
160
            tool_dicts = None if request.tools is None else [
                tool.model_dump() for tool in request.tools
            ]

161
162
163
164
165
166
167
168
169
            (
                conversation,
                request_prompts,
                engine_prompts,
            ) = await self._preprocess_chat(
                request,
                tokenizer,
                request.messages,
                chat_template=request.chat_template or self.chat_template,
170
                chat_template_content_format=self.chat_template_content_format,
171
172
173
174
175
176
177
178
179
180
181
                add_generation_prompt=request.add_generation_prompt,
                continue_final_message=request.continue_final_message,
                tool_dicts=tool_dicts,
                documents=request.documents,
                chat_template_kwargs=request.chat_template_kwargs,
                tool_parser=tool_parser,
                truncate_prompt_tokens=request.truncate_prompt_tokens,
                add_special_tokens=request.add_special_tokens,
            )
        except ValueError as e:
            logger.exception("Error in preprocessing prompt inputs")
182
183
            return self.create_error_response(str(e))

184
185
        request_id = "chatcmpl-" \
                     f"{self._base_request_id(raw_request, request.request_id)}"
186
187
188
189
190

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

191
192
        # Schedule the request and get the result generator.
        generators: List[AsyncGenerator[RequestOutput, None]] = []
193
        try:
194
195
196
197
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
                default_max_tokens = self.max_model_len - len(
                    engine_prompt["prompt_token_ids"])
198
199
200
                # Build default sampling params
                default_sampling_params = (
                    self.model_config.get_diff_sampling_param())
201
202
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
203
                        default_max_tokens, default_sampling_params)
204
205
                else:
                    sampling_params = request.to_sampling_params(
206
                        default_max_tokens,
207
208
                        self.model_config.logits_processor_pattern,
                        default_sampling_params)
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
                                 lora_request=lora_request,
                                 prompt_adapter_request=prompt_adapter_request)

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        prompt_adapter_request=prompt_adapter_request,
                        priority=request.priority,
                    )

                generators.append(generator)
237
        except ValueError as e:
238
            # TODO: Use a vllm-specific Validation Error
239
240
            return self.create_error_response(str(e))

241
242
243
        assert len(generators) == 1
        result_generator, = generators

244
245
246
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
247
248
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
249

250
251
        try:
            return await self.chat_completion_full_generator(
252
253
                request, result_generator, request_id, model_name,
                conversation, tokenizer, request_metadata)
254
255
256
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
257
258
259
260

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
261
        return request.messages[-1]["role"]
262
263

    async def chat_completion_stream_generator(
264
265
266
267
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
268
        model_name: str,
269
        conversation: List[ConversationMessage],
270
        tokenizer: AnyTokenizer,
271
        request_metadata: RequestResponseMetadata,
272
    ) -> AsyncGenerator[str, None]:
273
        created_time = int(time.time())
274
        chunk_object_type: Final = "chat.completion.chunk"
275
        first_iteration = True
276
277

        # Send response for each token for each request.n (index)
278
279
280
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
281
        num_prompt_tokens = 0
282
        num_cached_tokens = None
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

        all_previous_token_ids: Optional[List[List[int]]]
        if tool_choice_auto:
            # These are only required in "auto" tool choice case
            previous_texts = [""] * num_choices
            all_previous_token_ids = [[]] * num_choices
        else:
            previous_texts, all_previous_token_ids = None, None

302
303
304
305
306
307
308
309
310
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
                tool_parsers: List[Optional[ToolParser]] = [
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
        except RuntimeError as e:
311
            logger.exception("Error in tool parser creation.")
312
313
314
315
316
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

317
318
319
320
321
322
323
324
        stream_options = request.stream_options
        if stream_options:
            include_usage = stream_options.include_usage
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

325
326
        try:
            async for res in result_generator:
327
328
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
329
330
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
331

332
333
334
335
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
336
                    num_cached_tokens = res.num_cached_tokens
337
338
                    # Send first response for each request.n (index) with
                    # the role
339
                    role = self.get_chat_request_role(request)
340
341
342

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
343
                    for i in range(num_choices):
344
345
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
346
347
348
349
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
350
351
352
353
354
355
356
357
                            logprobs=None,
                            finish_reason=None)
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
                            model=model_name)
358

359
360
361
362
363
364
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
365

366
367
368
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

369
370
                    # Send response to echo the input portion of the
                    # last message
371
                    if request.echo:
372
                        last_msg_content: Union[str, List[Dict[str, str]]] = ""
373
374
375
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
376
377

                        if last_msg_content:
378
                            for i in range(num_choices):
379
380
381
382
383
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
384
                                        logprobs=None,
385
                                        finish_reason=None))
386
387
388
389
390
391
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
392
393
394
395
396
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
397

398
399
400
401
402
403
404
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
405
                    tool_parser = tool_parsers[i]
406
407
408
409

                    if finish_reason_sent[i]:
                        continue

410
                    if request.logprobs and request.top_logprobs is not None:
411
                        assert output.logprobs is not None, (
412
                            "Did not output logprobs")
413
                        logprobs = self._create_chat_logprobs(
414
415
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
416
                            tokenizer=tokenizer,
417
                            num_output_top_logprobs=request.top_logprobs,
418
419
420
421
                        )
                    else:
                        logprobs = None

422
                    delta_text = output.text
423
424
425
426
427
428

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

429
                    delta_message: Optional[DeltaMessage]
430

431
                    # handle streaming deltas for tools with named tool_choice
432
                    if tool_choice_function_name:
433
                        delta_message = DeltaMessage(tool_calls=[
434
                            DeltaToolCall(function=DeltaFunctionCall(
435
                                name=tool_choice_function_name,
436
437
                                arguments=delta_text),
                                          index=i)
438
                        ])
439
440

                    # handle streaming deltas for tools with "auto" tool choice
441
442
443
444
445
446
447
448
449
450
451
                    elif tool_choice_auto:
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        assert tool_parser is not None
                        #TODO optimize manipulation of these lists
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
                        current_token_ids = previous_token_ids + list(
                            output.token_ids)

452
453
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
454
455
                                previous_text=previous_text,
                                current_text=current_text,
456
                                delta_text=delta_text,
457
458
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
459
460
                                delta_token_ids=output.token_ids,
                                request=request))
461
462
463
464

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
465
466

                    # handle streaming just a content delta
467
468
469
                    else:
                        delta_message = DeltaMessage(content=delta_text)

470
                    # set the previous values for the next iteration
471
                    previous_num_tokens[i] += len(output.token_ids)
472
473
474
475
476
477
478
479

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

480
481
482
483
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
484
                            delta=delta_message,
485
486
                            logprobs=logprobs,
                            finish_reason=None)
487
488

                    # if the model is finished generating
489
                    else:
490
491
492
493
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
494
                        auto_tools_called = False
495
                        if tool_parser:
496
497
498
499
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
500
501
502
503
504
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
505
506
507
508
509
510
511
512
513
514
                            latest_delta_len = 0
                            if ((isinstance(
                                    delta_message.tool_calls[0].function,
                                    DeltaFunctionCall)) and isinstance(
                                        delta_message.tool_calls[0].function.
                                        arguments, str)):
                                latest_delta_len = len(
                                    delta_message.tool_calls[0].function.
                                    arguments)

515
516
517
518
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
519
520
                                    "arguments", {}),
                                ensure_ascii=False)
521

522
                            # get what we've streamed so far for arguments
523
524
525
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]
526
527
                            if (latest_delta_len > 0):
                                actual_call = actual_call[:-latest_delta_len]
528
529
530
531
532
533
534
535
536
537
538
539

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)
                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

540
541
542
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
543
                            delta=delta_message,
544
                            logprobs=logprobs,
545
                            finish_reason=output.finish_reason
546
                            if not auto_tools_called else "tool_calls",
547
                            stop_reason=output.stop_reason)
548

549
                        finish_reason_sent[i] = True
550

551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

                    data = chunk.model_dump_json(exclude_unset=True)
                    yield f"data: {data}\n\n"

570
571
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
572
573
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
574
575
576
577
578
579
580
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
581
582
583
584
585
586
587
588
589
590
591

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
592

593
594
595
596
597
598
599
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
                total_tokens=num_prompt_tokens + num_completion_tokens)

600
601
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
602
            logger.exception("Error in chat completion stream generator.")
603
604
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
605
606
607
608
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
609
610
611
612
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
613
        model_name: str,
614
        conversation: List[ConversationMessage],
615
        tokenizer: AnyTokenizer,
616
        request_metadata: RequestResponseMetadata,
617
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
618

619
        created_time = int(time.time())
620
        final_res: Optional[RequestOutput] = None
621

622
623
624
625
626
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
627
628
629
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
630

631
632
        assert final_res is not None

633
        choices: List[ChatCompletionResponseChoice] = []
634

635
636
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
637
            token_ids = output.token_ids
638
            out_logprobs = output.logprobs
639

640
641
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
642
                logprobs = self._create_chat_logprobs(
643
                    token_ids=token_ids,
644
                    top_logprobs=out_logprobs,
645
                    num_output_top_logprobs=request.top_logprobs,
646
                    tokenizer=tokenizer,
647
648
649
650
                )
            else:
                logprobs = None

651
652
653
654
            # In the OpenAI API the finish_reason is "tools_called"
            # if the tool choice is auto and the model produced a tool
            # call. The same is not true for named function calls
            auto_tools_called = False
655
656
657

            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
658
            if (not self.enable_auto_tools
659
660
661
662
663
664
665
                    or not self.tool_parser) and not isinstance(
                        request.tool_choice,
                        ChatCompletionNamedToolChoiceParam):
                message = ChatMessage(role=role, content=output.text)

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
666
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
667

668
669
670
671
672
673
674
675
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
                        ToolCall(function=FunctionCall(
                            name=request.tool_choice.function.name,
                            arguments=output.text))
                    ])
676
677
678

            # if the request doesn't use tool choice
            # OR specifies to not use a tool
679
            elif not request.tool_choice or request.tool_choice == "none":
680
681
682
683
684
685
686
687
688

                message = ChatMessage(role=role, content=output.text)

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

689
690
691
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
692
                    logger.exception("Error in tool parser creation.")
693
694
                    return self.create_error_response(str(e))

695
696
                tool_call_info = tool_parser.extract_tool_calls(
                    output.text, request=request)
697
698
699
700
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
                    message = ChatMessage(role=role, content=output.text)

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
717
718
                message = ChatMessage(role=role, content=output.text)

719
720
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
721
                message=message,
722
                logprobs=logprobs,
723
                finish_reason="tool_calls" if auto_tools_called else
724
                output.finish_reason if output.finish_reason else "stop",
725
                stop_reason=output.stop_reason)
726
727
            choices.append(choice_data)

728
        if request.echo:
729
            last_msg_content: Union[str, List[Dict[str, str]]] = ""
730
731
            if conversation and "content" in conversation[-1] and conversation[
                    -1].get("role") == role:
732
                last_msg_content = conversation[-1]["content"] or ""
733
734
735
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
736
737

            for choice in choices:
738
739
                full_message = last_msg_content + (choice.message.content
                                                   or "")
740
741
                choice.message.content = full_message

742
        assert final_res.prompt_token_ids is not None
743
        num_prompt_tokens = len(final_res.prompt_token_ids)
744
745
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
746
747
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
748
749
750
751
752
753
754
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
755
756
757

        request_metadata.final_usage_info = usage

758
759
760
761
762
763
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
764
            prompt_logprobs=final_res.prompt_logprobs,
765
766
        )

767
        return response
768
769

    def _get_top_logprobs(
770
            self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
771
            tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]:
772
        return [
773
774
775
776
777
778
779
780
            ChatCompletionLogProb(token=(token := self._get_decoded_token(
                p[1],
                p[0],
                tokenizer,
                return_as_token_id=self.return_tokens_as_token_ids)),
                                  logprob=max(p[1].logprob, -9999.0),
                                  bytes=list(
                                      token.encode("utf-8", errors="replace")))
781
782
783
784
785
786
787
788
            for i, p in enumerate(logprobs.items())
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
        top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
789
        tokenizer: AnyTokenizer,
790
791
792
        num_output_top_logprobs: Optional[int] = None,
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
793
        logprobs_content: List[ChatCompletionLogProbsContent] = []
794
795
796
797

        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
            if step_top_logprobs is None:
798
                token = tokenizer.decode(token_id)
799
800
                if self.return_tokens_as_token_ids:
                    token = f"token_id:{token_id}"
801

802
803
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
804
                        token=token,
805
806
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
807
            else:
808
809
810
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

811
812
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
813
                        token=self._get_decoded_token(
814
815
816
817
818
819
820
821
                            step_token,
                            token_id,
                            tokenizer,
                            self.return_tokens_as_token_ids,
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
822
                        top_logprobs=self._get_top_logprobs(
823
824
825
826
827
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                        ),
                    ))
828
829

        return ChatCompletionLogProbs(content=logprobs_content)
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
859
860
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
861
862
863
864
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )