serving_chat.py 39.1 KB
Newer Older
1
import asyncio
2
import json
3
import time
4
5
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, Final, List,
                    Optional)
6
from typing import Sequence as GenericSequence
7
from typing import Union
8

9
from fastapi import Request
10

11
from vllm.config import ModelConfig
12
from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.chat_utils import ConversationMessage, load_chat_template
14
from vllm.entrypoints.logger import RequestLogger
15
from vllm.entrypoints.openai.protocol import (
16
17
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
18
    ChatCompletionRequest, ChatCompletionResponse,
19
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
20
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
21
22
    DeltaToolCall, ErrorResponse, FunctionCall, PromptTokenUsageInfo,
    RequestResponseMetadata, ToolCall, UsageInfo)
23
24
from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
                                                    LoRAModulePath,
25
                                                    OpenAIServing,
26
                                                    PromptAdapterPath)
27
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
28
from vllm.logger import init_logger
29
from vllm.outputs import CompletionOutput, RequestOutput
30
from vllm.sampling_params import BeamSearchParams, SamplingParams
31
from vllm.sequence import Logprob
32
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
33
from vllm.utils import iterate_with_cancellation
34
35
36
37
38
39

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

40
    def __init__(self,
41
                 engine_client: EngineClient,
42
                 model_config: ModelConfig,
43
                 base_model_paths: List[BaseModelPath],
44
45
46
47
48
49
50
51
                 response_role: str,
                 *,
                 lora_modules: Optional[List[LoRAModulePath]],
                 prompt_adapters: Optional[List[PromptAdapterPath]],
                 request_logger: Optional[RequestLogger],
                 chat_template: Optional[str],
                 return_tokens_as_token_ids: bool = False,
                 enable_auto_tools: bool = False,
52
53
                 tool_parser: Optional[str] = None,
                 enable_prompt_tokens_details: bool = False):
54
        super().__init__(engine_client=engine_client,
55
                         model_config=model_config,
56
                         base_model_paths=base_model_paths,
57
58
                         lora_modules=lora_modules,
                         prompt_adapters=prompt_adapters,
59
60
                         request_logger=request_logger,
                         return_tokens_as_token_ids=return_tokens_as_token_ids)
61

62
        self.response_role = response_role
63
        self.use_tool_use_model_template = False
64
        self.chat_template = load_chat_template(chat_template)
65

66
67
68
69
70
71
72
73
74
75
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
76
            try:
77
78
79
80
81
                if (tool_parser == "pythonic" and
                        model_config.model.startswith("meta-llama/Llama-3.2")):
                    logger.warning(
                        "Llama3.2 models may struggle to emit valid pythonic"
                        " tool calls")
82
83
84
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
85
                raise TypeError("Error: --enable-auto-tool-choice requires "
86
87
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
88

89
90
        self.enable_prompt_tokens_details = enable_prompt_tokens_details

91
    async def create_chat_completion(
92
93
        self,
        request: ChatCompletionRequest,
94
95
96
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
97
98
        """
        Chat Completion API similar to OpenAI's API.
99

100
101
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
102
        Chat Completion API.
103
104
105
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
106
            logger.error("Error with model %s", error_check_ret)
107
108
            return error_check_ret

109
110
111
112
113
114
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

115
        try:
116
117
118
119
120
            (
                lora_request,
                prompt_adapter_request,
            ) = self._maybe_get_adapters(request)

121
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
122
123
124
125
126
127
128
129
            tool_parser = self.tool_parser

            # validation for OpenAI tools
            # tool_choice = "required" is not supported
            if request.tool_choice == "required":
                return self.create_error_response(
                    "tool_choice = \"required\" is not supported!")

130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
            # NOTE: There is currently a bug in pydantic where attributes
            # declared as iterables are replaced in in the instances by
            # pydantic-core ValidatorIterator instance. In particular, this
            # affects tool_calls defined in ChatCompletionAssistantMessageParam
            # model:
            # see:
            #   - https://github.com/pydantic/pydantic/issues/9467
            # As a result, tool_calls from assistant messages are never
            # deserialized in the request object if the tool_calls iterator is
            # not consumed. This affect messages passed to the MistralTokenizer
            # since no chat template is applied and therefore the tools_calls
            # iterator is not directly consumed.
            # Issue is tracked on Pydantic side, with resolution planned for
            # v2.11 release. In the meantime, the official workaround is to
            # consume the iterator so the tool_calls are correctly deserialized
            # in the OpenAI ChatCompletionAssistantMessageParam object
            # https://github.com/pydantic/pydantic/issues/9467#issuecomment-2442097291 # noqa: E501
            # Official Pydantic Issues:
            #   - https://github.com/pydantic/pydantic/issues/9541
            # TODO: remove when pydantic v2.11 is released
            if isinstance(tokenizer, MistralTokenizer):
                for i, message in enumerate(request.messages):
                    if message.get("role") == 'assistant':
                        tool_calls_validator = message.get(
                            "tool_calls", ().__iter__())
                        validated_tool_calls = []
                        while True:
                            try:
                                tool_call = next(
                                    tool_calls_validator)  # type: ignore
                                validated_tool_calls.append(tool_call)
                            except StopIteration:
                                break
                        request.messages[i][
                            "tool_calls"] = validated_tool_calls

166
167
168
169
170
171
172
173
174
            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
                    and not isinstance(tokenizer, MistralTokenizer)):
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
175

176
177
178
179
            tool_dicts = None if request.tools is None else [
                tool.model_dump() for tool in request.tools
            ]

180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
            (
                conversation,
                request_prompts,
                engine_prompts,
            ) = await self._preprocess_chat(
                request,
                tokenizer,
                request.messages,
                chat_template=request.chat_template or self.chat_template,
                add_generation_prompt=request.add_generation_prompt,
                continue_final_message=request.continue_final_message,
                tool_dicts=tool_dicts,
                documents=request.documents,
                chat_template_kwargs=request.chat_template_kwargs,
                tool_parser=tool_parser,
                truncate_prompt_tokens=request.truncate_prompt_tokens,
                add_special_tokens=request.add_special_tokens,
            )
        except ValueError as e:
            logger.exception("Error in preprocessing prompt inputs")
200
201
            return self.create_error_response(str(e))

202
        request_id = f"chatcmpl-{request.request_id}"
203
204
205
206
207

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

208
209
        # Schedule the request and get the result generator.
        generators: List[AsyncGenerator[RequestOutput, None]] = []
210
        try:
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
                default_max_tokens = self.max_model_len - len(
                    engine_prompt["prompt_token_ids"])
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
                        default_max_tokens)
                else:
                    sampling_params = request.to_sampling_params(
                        default_max_tokens)

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
                                 lora_request=lora_request,
                                 prompt_adapter_request=prompt_adapter_request)

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        request_id=request_id,
                        params=sampling_params,
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        prompt_adapter_request=prompt_adapter_request,
                        priority=request.priority,
                    )

                generators.append(generator)
249
        except ValueError as e:
250
            # TODO: Use a vllm-specific Validation Error
251
252
            return self.create_error_response(str(e))

253
254
255
        assert len(generators) == 1
        result_generator, = generators

256
257
258
259
        if raw_request:
            result_generator = iterate_with_cancellation(
                result_generator, raw_request.is_disconnected)

260
261
262
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
263
264
                request, result_generator, request_id, conversation, tokenizer,
                request_metadata)
265

266
267
        try:
            return await self.chat_completion_full_generator(
268
269
                request, result_generator, request_id, conversation, tokenizer,
                request_metadata)
270
271
272
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
273
274
275
276

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
277
        return request.messages[-1]["role"]
278
279

    async def chat_completion_stream_generator(
280
281
282
283
284
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
        conversation: List[ConversationMessage],
285
        tokenizer: AnyTokenizer,
286
        request_metadata: RequestResponseMetadata,
287
    ) -> AsyncGenerator[str, None]:
288
        model_name = self.base_model_paths[0].name
289
        created_time = int(time.time())
290
        chunk_object_type: Final = "chat.completion.chunk"
291
        first_iteration = True
292
293

        # Send response for each token for each request.n (index)
294
295
296
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
297
        num_prompt_tokens = 0
298
        num_cached_tokens = None
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

        all_previous_token_ids: Optional[List[List[int]]]
        if tool_choice_auto:
            # These are only required in "auto" tool choice case
            previous_texts = [""] * num_choices
            all_previous_token_ids = [[]] * num_choices
        else:
            previous_texts, all_previous_token_ids = None, None

318
319
320
321
322
323
324
325
326
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
                tool_parsers: List[Optional[ToolParser]] = [
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
        except RuntimeError as e:
327
            logger.exception("Error in tool parser creation.")
328
329
330
331
332
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

333
334
335
336
337
338
339
340
        stream_options = request.stream_options
        if stream_options:
            include_usage = stream_options.include_usage
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

341
342
        try:
            async for res in result_generator:
343
344
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
345
346
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
347

348
349
350
351
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
352
                    num_cached_tokens = res.num_cached_tokens
353
354
                    # Send first response for each request.n (index) with
                    # the role
355
                    role = self.get_chat_request_role(request)
356
357
358

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
359
                    for i in range(num_choices):
360
361
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
362
363
364
365
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
366
367
368
369
370
371
372
373
                            logprobs=None,
                            finish_reason=None)
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
                            model=model_name)
374

375
376
377
378
379
380
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
381

382
383
384
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

385
386
                    # Send response to echo the input portion of the
                    # last message
387
                    if request.echo or request.continue_final_message:
388
                        last_msg_content: Union[str, List[Dict[str, str]]] = ""
389
390
391
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
392
393

                        if last_msg_content:
394
                            for i in range(num_choices):
395
396
397
398
399
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
400
                                        logprobs=None,
401
                                        finish_reason=None))
402
403
404
405
406
407
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
408
409
410
411
412
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
413

414
415
416
417
418
419
420
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
421
                    tool_parser = tool_parsers[i]
422
423
424
425

                    if finish_reason_sent[i]:
                        continue

426
                    if request.logprobs and request.top_logprobs is not None:
427
                        assert output.logprobs is not None, (
428
                            "Did not output logprobs")
429
                        logprobs = self._create_chat_logprobs(
430
431
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
432
                            tokenizer=tokenizer,
433
                            num_output_top_logprobs=request.top_logprobs,
434
435
436
437
                        )
                    else:
                        logprobs = None

438
                    delta_text = output.text
439
440
441
442
443
444

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

445
                    delta_message: Optional[DeltaMessage]
446

447
                    # handle streaming deltas for tools with named tool_choice
448
                    if tool_choice_function_name:
449
                        delta_message = DeltaMessage(tool_calls=[
450
                            DeltaToolCall(function=DeltaFunctionCall(
451
                                name=tool_choice_function_name,
452
453
                                arguments=delta_text),
                                          index=i)
454
                        ])
455
456

                    # handle streaming deltas for tools with "auto" tool choice
457
458
459
460
461
462
463
464
465
466
467
                    elif tool_choice_auto:
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        assert tool_parser is not None
                        #TODO optimize manipulation of these lists
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
                        current_token_ids = previous_token_ids + list(
                            output.token_ids)

468
469
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
470
471
                                previous_text=previous_text,
                                current_text=current_text,
472
                                delta_text=delta_text,
473
474
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
475
476
                                delta_token_ids=output.token_ids,
                                request=request))
477
478
479
480

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
481
482

                    # handle streaming just a content delta
483
484
485
                    else:
                        delta_message = DeltaMessage(content=delta_text)

486
                    # set the previous values for the next iteration
487
                    previous_num_tokens[i] += len(output.token_ids)
488
489
490
491
492
493
494
495

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

496
497
498
499
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
500
                            delta=delta_message,
501
502
                            logprobs=logprobs,
                            finish_reason=None)
503
504

                    # if the model is finished generating
505
                    else:
506
507
508
509
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
510
                        auto_tools_called = False
511
                        if tool_parser:
512
513
514
515
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
516
517
518
519
520
521
522
523
524
525
526
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
                                    "arguments", {}))

527
                            # get what we've streamed so far for arguments
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)

                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

544
545
546
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
547
                            delta=delta_message,
548
                            logprobs=logprobs,
549
                            finish_reason=output.finish_reason
550
                            if not auto_tools_called else "tool_calls",
551
                            stop_reason=output.stop_reason)
552

553
                        finish_reason_sent[i] = True
554

555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

                    data = chunk.model_dump_json(exclude_unset=True)
                    yield f"data: {data}\n\n"

574
575
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
576
577
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
578
579
580
581
582
583
584
                final_usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                                        completion_tokens=completion_tokens,
                                        total_tokens=num_prompt_tokens +
                                        completion_tokens)
                if self.enable_prompt_tokens_details and num_cached_tokens:
                    final_usage.prompt_tokens_details = PromptTokenUsageInfo(
                        cached_tokens=num_cached_tokens)
585
586
587
588
589
590
591
592
593
594
595

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
596

597
598
599
600
601
602
603
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
                total_tokens=num_prompt_tokens + num_completion_tokens)

604
605
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
606
            logger.exception("Error in chat completion stream generator.")
607
608
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
609
610
611
612
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
613
614
615
616
617
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
        conversation: List[ConversationMessage],
618
        tokenizer: AnyTokenizer,
619
        request_metadata: RequestResponseMetadata,
620
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
621

622
        model_name = self.base_model_paths[0].name
623
        created_time = int(time.time())
624
        final_res: Optional[RequestOutput] = None
625

626
627
628
629
630
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
631
632
633
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
634

635
636
        assert final_res is not None

637
        choices: List[ChatCompletionResponseChoice] = []
638

639
640
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
641
            token_ids = output.token_ids
642
            out_logprobs = output.logprobs
643

644
645
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
646
                logprobs = self._create_chat_logprobs(
647
                    token_ids=token_ids,
648
                    top_logprobs=out_logprobs,
649
                    num_output_top_logprobs=request.top_logprobs,
650
                    tokenizer=tokenizer,
651
652
653
654
                )
            else:
                logprobs = None

655
656
657
658
            # In the OpenAI API the finish_reason is "tools_called"
            # if the tool choice is auto and the model produced a tool
            # call. The same is not true for named function calls
            auto_tools_called = False
659
660
661

            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
662
            if (not self.enable_auto_tools
663
664
665
666
667
668
669
                    or not self.tool_parser) and not isinstance(
                        request.tool_choice,
                        ChatCompletionNamedToolChoiceParam):
                message = ChatMessage(role=role, content=output.text)

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
670
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
671

672
673
674
675
676
677
678
679
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
                        ToolCall(function=FunctionCall(
                            name=request.tool_choice.function.name,
                            arguments=output.text))
                    ])
680
681
682

            # if the request doesn't use tool choice
            # OR specifies to not use a tool
683
            elif not request.tool_choice or request.tool_choice == "none":
684
685
686
687
688
689
690
691
692

                message = ChatMessage(role=role, content=output.text)

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

693
694
695
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
696
                    logger.exception("Error in tool parser creation.")
697
698
                    return self.create_error_response(str(e))

699
700
                tool_call_info = tool_parser.extract_tool_calls(
                    output.text, request=request)
701
702
703
704
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
                    message = ChatMessage(role=role, content=output.text)

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
721
722
                message = ChatMessage(role=role, content=output.text)

723
724
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
725
                message=message,
726
                logprobs=logprobs,
727
                finish_reason="tool_calls" if auto_tools_called else
728
                output.finish_reason if output.finish_reason else "stop",
729
                stop_reason=output.stop_reason)
730
731
            choices.append(choice_data)

732
        if request.echo or request.continue_final_message:
733
            last_msg_content: Union[str, List[Dict[str, str]]] = ""
734
735
            if conversation and "content" in conversation[-1] and conversation[
                    -1].get("role") == role:
736
                last_msg_content = conversation[-1]["content"] or ""
737
738
739
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
740
741

            for choice in choices:
742
743
                full_message = last_msg_content + (choice.message.content
                                                   or "")
744
745
                choice.message.content = full_message

746
        assert final_res.prompt_token_ids is not None
747
        num_prompt_tokens = len(final_res.prompt_token_ids)
748
749
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
750
751
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
752
753
754
755
756
757
758
        usage = UsageInfo(prompt_tokens=num_prompt_tokens,
                          completion_tokens=num_generated_tokens,
                          total_tokens=num_prompt_tokens +
                          num_generated_tokens)
        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
            usage.prompt_tokens_details = PromptTokenUsageInfo(
                cached_tokens=final_res.num_cached_tokens)
759
760
761

        request_metadata.final_usage_info = usage

762
763
764
765
766
767
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
768
            prompt_logprobs=final_res.prompt_logprobs,
769
770
        )

771
        return response
772
773

    def _get_top_logprobs(
774
            self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
775
            tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]:
776
        return [
777
778
779
780
781
782
783
784
            ChatCompletionLogProb(token=(token := self._get_decoded_token(
                p[1],
                p[0],
                tokenizer,
                return_as_token_id=self.return_tokens_as_token_ids)),
                                  logprob=max(p[1].logprob, -9999.0),
                                  bytes=list(
                                      token.encode("utf-8", errors="replace")))
785
786
787
788
789
790
791
792
            for i, p in enumerate(logprobs.items())
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
        top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
793
        tokenizer: AnyTokenizer,
794
795
796
        num_output_top_logprobs: Optional[int] = None,
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
797
        logprobs_content: List[ChatCompletionLogProbsContent] = []
798
799
800
801

        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
            if step_top_logprobs is None:
802
                token = tokenizer.decode(token_id)
803
804
                if self.return_tokens_as_token_ids:
                    token = f"token_id:{token_id}"
805

806
807
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
808
                        token=token,
809
810
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
811
            else:
812
813
814
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

815
816
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
817
                        token=self._get_decoded_token(
818
819
820
821
822
823
824
825
                            step_token,
                            token_id,
                            tokenizer,
                            self.return_tokens_as_token_ids,
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
826
                        top_logprobs=self._get_top_logprobs(
827
828
829
830
831
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                        ),
                    ))
832
833

        return ChatCompletionLogProbs(content=logprobs_content)
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
863
864
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
865
866
867
868
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )