serving_chat.py 36 KB
Newer Older
1
import asyncio
2
import json
3
import time
4
5
from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, Final, List,
                    Optional)
6
from typing import Sequence as GenericSequence
7
from typing import Union
8

9
from fastapi import Request
10

11
from vllm.config import ModelConfig
12
from vllm.engine.protocol import EngineClient
13
from vllm.entrypoints.chat_utils import ConversationMessage, load_chat_template
14
from vllm.entrypoints.logger import RequestLogger
15
from vllm.entrypoints.openai.protocol import (
16
17
    ChatCompletionLogProb, ChatCompletionLogProbs,
    ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam,
18
    ChatCompletionRequest, ChatCompletionResponse,
19
    ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice,
20
    ChatCompletionStreamResponse, ChatMessage, DeltaFunctionCall, DeltaMessage,
21
22
    DeltaToolCall, ErrorResponse, FunctionCall, RequestResponseMetadata,
    ToolCall, UsageInfo)
23
24
from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
                                                    LoRAModulePath,
25
                                                    OpenAIServing,
26
                                                    PromptAdapterPath)
27
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
28
from vllm.logger import init_logger
29
from vllm.outputs import CompletionOutput, RequestOutput
30
from vllm.sampling_params import BeamSearchParams, SamplingParams
31
from vllm.sequence import Logprob
32
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
33
from vllm.utils import iterate_with_cancellation
34
35
36
37
38
39

logger = init_logger(__name__)


class OpenAIServingChat(OpenAIServing):

40
    def __init__(self,
41
                 engine_client: EngineClient,
42
                 model_config: ModelConfig,
43
                 base_model_paths: List[BaseModelPath],
44
45
46
47
48
49
50
51
52
                 response_role: str,
                 *,
                 lora_modules: Optional[List[LoRAModulePath]],
                 prompt_adapters: Optional[List[PromptAdapterPath]],
                 request_logger: Optional[RequestLogger],
                 chat_template: Optional[str],
                 return_tokens_as_token_ids: bool = False,
                 enable_auto_tools: bool = False,
                 tool_parser: Optional[str] = None):
53
        super().__init__(engine_client=engine_client,
54
                         model_config=model_config,
55
                         base_model_paths=base_model_paths,
56
57
                         lora_modules=lora_modules,
                         prompt_adapters=prompt_adapters,
58
59
                         request_logger=request_logger,
                         return_tokens_as_token_ids=return_tokens_as_token_ids)
60

61
        self.response_role = response_role
62
        self.use_tool_use_model_template = False
63
        self.chat_template = load_chat_template(chat_template)
64

65
66
67
68
69
70
71
72
73
74
        # set up tool use
        self.enable_auto_tools: bool = enable_auto_tools
        if self.enable_auto_tools:
            logger.info(
                "\"auto\" tool choice has been enabled please note that while"
                " the parallel_tool_calls client option is preset for "
                "compatibility reasons, it will be ignored.")

        self.tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None
        if self.enable_auto_tools:
75
76
77
78
            try:
                self.tool_parser = ToolParserManager.get_tool_parser(
                    tool_parser)
            except Exception as e:
79
                raise TypeError("Error: --enable-auto-tool-choice requires "
80
81
                                f"tool_parser:'{tool_parser}' which has not "
                                "been registered") from e
82

83
    async def create_chat_completion(
84
85
        self,
        request: ChatCompletionRequest,
86
87
88
        raw_request: Optional[Request] = None,
    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
               ErrorResponse]:
89
90
        """
        Chat Completion API similar to OpenAI's API.
91

92
93
        See https://platform.openai.com/docs/api-reference/chat/create
        for the API specification. This API mimics the OpenAI
94
        Chat Completion API.
95
96
97
        """
        error_check_ret = await self._check_model(request)
        if error_check_ret is not None:
98
            logger.error("Error with model %s", error_check_ret)
99
100
            return error_check_ret

101
102
103
104
105
106
        # If the engine is dead, raise the engine's DEAD_ERROR.
        # This is required for the streaming case, where we return a
        # success status before we actually start generating text :).
        if self.engine_client.errored:
            raise self.engine_client.dead_error

107
        try:
108
109
110
111
112
            (
                lora_request,
                prompt_adapter_request,
            ) = self._maybe_get_adapters(request)

113
            tokenizer = await self.engine_client.get_tokenizer(lora_request)
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
            tool_parser = self.tool_parser

            # validation for OpenAI tools
            # tool_choice = "required" is not supported
            if request.tool_choice == "required":
                return self.create_error_response(
                    "tool_choice = \"required\" is not supported!")

            if (request.tool_choice == "auto" and
                    not (self.enable_auto_tools and tool_parser is not None)
                    and not isinstance(tokenizer, MistralTokenizer)):
                # for hf tokenizers, "auto" tools requires
                # --enable-auto-tool-choice and --tool-call-parser
                return self.create_error_response(
                    "\"auto\" tool choice requires "
                    "--enable-auto-tool-choice and --tool-call-parser to be set"
                )
131

132
133
134
135
            tool_dicts = None if request.tools is None else [
                tool.model_dump() for tool in request.tools
            ]

136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
            (
                conversation,
                request_prompts,
                engine_prompts,
            ) = await self._preprocess_chat(
                request,
                tokenizer,
                request.messages,
                chat_template=request.chat_template or self.chat_template,
                add_generation_prompt=request.add_generation_prompt,
                continue_final_message=request.continue_final_message,
                tool_dicts=tool_dicts,
                documents=request.documents,
                chat_template_kwargs=request.chat_template_kwargs,
                tool_parser=tool_parser,
                truncate_prompt_tokens=request.truncate_prompt_tokens,
                add_special_tokens=request.add_special_tokens,
            )
        except ValueError as e:
            logger.exception("Error in preprocessing prompt inputs")
156
157
            return self.create_error_response(str(e))

158
        request_id = f"chatcmpl-{request.request_id}"
159
160
161
162
163

        request_metadata = RequestResponseMetadata(request_id=request_id)
        if raw_request:
            raw_request.state.request_metadata = request_metadata

164
165
        # Schedule the request and get the result generator.
        generators: List[AsyncGenerator[RequestOutput, None]] = []
166
        try:
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
            for i, engine_prompt in enumerate(engine_prompts):
                sampling_params: Union[SamplingParams, BeamSearchParams]
                default_max_tokens = self.max_model_len - len(
                    engine_prompt["prompt_token_ids"])
                if request.use_beam_search:
                    sampling_params = request.to_beam_search_params(
                        default_max_tokens)
                else:
                    sampling_params = request.to_sampling_params(
                        default_max_tokens)

                self._log_inputs(request_id,
                                 request_prompts[i],
                                 params=sampling_params,
                                 lora_request=lora_request,
                                 prompt_adapter_request=prompt_adapter_request)

                trace_headers = (None if raw_request is None else await
                                 self._get_trace_headers(raw_request.headers))

                if isinstance(sampling_params, BeamSearchParams):
                    generator = self.engine_client.beam_search(
                        prompt=engine_prompt,
                        model_config=self.model_config,
                        request_id=request_id,
                        params=sampling_params,
                    )
                else:
                    generator = self.engine_client.generate(
                        engine_prompt,
                        sampling_params,
                        request_id,
                        lora_request=lora_request,
                        trace_headers=trace_headers,
                        prompt_adapter_request=prompt_adapter_request,
                        priority=request.priority,
                    )

                generators.append(generator)
206
        except ValueError as e:
207
            # TODO: Use a vllm-specific Validation Error
208
209
            return self.create_error_response(str(e))

210
211
212
        assert len(generators) == 1
        result_generator, = generators

213
214
215
216
        if raw_request:
            result_generator = iterate_with_cancellation(
                result_generator, raw_request.is_disconnected)

217
218
219
        # Streaming response
        if request.stream:
            return self.chat_completion_stream_generator(
220
221
                request, result_generator, request_id, conversation, tokenizer,
                request_metadata)
222

223
224
        try:
            return await self.chat_completion_full_generator(
225
226
                request, result_generator, request_id, conversation, tokenizer,
                request_metadata)
227
228
229
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
230
231
232
233

    def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
        if request.add_generation_prompt:
            return self.response_role
234
        return request.messages[-1]["role"]
235
236

    async def chat_completion_stream_generator(
237
238
239
240
241
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
        conversation: List[ConversationMessage],
242
        tokenizer: AnyTokenizer,
243
        request_metadata: RequestResponseMetadata,
244
    ) -> AsyncGenerator[str, None]:
245
        model_name = self.base_model_paths[0].name
246
        created_time = int(time.time())
247
        chunk_object_type: Final = "chat.completion.chunk"
248
        first_iteration = True
249
250

        # Send response for each token for each request.n (index)
251
252
253
        num_choices = 1 if request.n is None else request.n
        previous_num_tokens = [0] * num_choices
        finish_reason_sent = [False] * num_choices
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
        num_prompt_tokens = 0

        if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
            tool_choice_function_name = request.tool_choice.function.name
        else:
            tool_choice_function_name = None

        # Determine whether tools are in use with "auto" tool choice
        tool_choice_auto = (
            not tool_choice_function_name
            and self._should_stream_with_auto_tool_parsing(request))

        all_previous_token_ids: Optional[List[List[int]]]
        if tool_choice_auto:
            # These are only required in "auto" tool choice case
            previous_texts = [""] * num_choices
            all_previous_token_ids = [[]] * num_choices
        else:
            previous_texts, all_previous_token_ids = None, None

274
275
276
277
278
279
280
281
282
        # Prepare the tool parser if it's needed
        try:
            if tool_choice_auto and self.tool_parser:
                tool_parsers: List[Optional[ToolParser]] = [
                    self.tool_parser(tokenizer)
                ] * num_choices
            else:
                tool_parsers = [None] * num_choices
        except RuntimeError as e:
283
            logger.exception("Error in tool parser creation.")
284
285
286
287
288
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"
            return

289
290
291
292
293
294
295
296
        stream_options = request.stream_options
        if stream_options:
            include_usage = stream_options.include_usage
            include_continuous_usage = include_usage and \
                                       stream_options.continuous_usage_stats
        else:
            include_usage, include_continuous_usage = False, False

297
298
        try:
            async for res in result_generator:
299
300
                if res.prompt_token_ids is not None:
                    num_prompt_tokens = len(res.prompt_token_ids)
301
302
                    if res.encoder_prompt_token_ids is not None:
                        num_prompt_tokens += len(res.encoder_prompt_token_ids)
303

304
305
306
307
                # We need to do it here, because if there are exceptions in
                # the result_generator, it needs to be sent as the FIRST
                # response (by the try...catch).
                if first_iteration:
308
309
                    # Send first response for each request.n (index) with
                    # the role
310
                    role = self.get_chat_request_role(request)
311
312
313

                    # NOTE num_choices defaults to 1 so this usually executes
                    # once per request
314
                    for i in range(num_choices):
315
316
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
317
318
319
320
                            delta=DeltaMessage(
                                role=role,
                                content="",
                            ),
321
322
323
324
325
326
327
328
                            logprobs=None,
                            finish_reason=None)
                        chunk = ChatCompletionStreamResponse(
                            id=request_id,
                            object=chunk_object_type,
                            created=created_time,
                            choices=[choice_data],
                            model=model_name)
329

330
331
332
333
334
335
                        # if continuous usage stats are requested, add it
                        if include_continuous_usage:
                            chunk.usage = UsageInfo(
                                prompt_tokens=num_prompt_tokens,
                                completion_tokens=0,
                                total_tokens=num_prompt_tokens)
336

337
338
339
                        data = chunk.model_dump_json(exclude_unset=True)
                        yield f"data: {data}\n\n"

340
341
                    # Send response to echo the input portion of the
                    # last message
342
                    if request.echo or request.continue_final_message:
343
                        last_msg_content: Union[str, List[Dict[str, str]]] = ""
344
345
346
                        if conversation and "content" in conversation[
                                -1] and conversation[-1].get("role") == role:
                            last_msg_content = conversation[-1]["content"] or ""
347
348

                        if last_msg_content:
349
                            for i in range(num_choices):
350
351
352
353
354
                                choice_data = (
                                    ChatCompletionResponseStreamChoice(
                                        index=i,
                                        delta=DeltaMessage(
                                            content=last_msg_content),
355
                                        logprobs=None,
356
                                        finish_reason=None))
357
358
359
360
361
362
                                chunk = ChatCompletionStreamResponse(
                                    id=request_id,
                                    object=chunk_object_type,
                                    created=created_time,
                                    choices=[choice_data],
                                    model=model_name)
363
364
365
366
367
                                if include_continuous_usage:
                                    chunk.usage = UsageInfo(
                                        prompt_tokens=num_prompt_tokens,
                                        completion_tokens=0,
                                        total_tokens=num_prompt_tokens)
368

369
370
371
372
373
374
375
                                data = chunk.model_dump_json(
                                    exclude_unset=True)
                                yield f"data: {data}\n\n"
                    first_iteration = False

                for output in res.outputs:
                    i = output.index
376
                    tool_parser = tool_parsers[i]
377
378
379
380

                    if finish_reason_sent[i]:
                        continue

381
                    if request.logprobs and request.top_logprobs is not None:
382
                        assert output.logprobs is not None, (
383
                            "Did not output logprobs")
384
                        logprobs = self._create_chat_logprobs(
385
386
                            token_ids=output.token_ids,
                            top_logprobs=output.logprobs,
387
                            tokenizer=tokenizer,
388
                            num_output_top_logprobs=request.top_logprobs,
389
390
391
392
                        )
                    else:
                        logprobs = None

393
                    delta_text = output.text
394
395
396
397
398
399

                    if not delta_text and not output.token_ids and \
                        not previous_num_tokens[i]:
                        # Chunked prefill case, don't return empty chunks
                        continue

400
                    delta_message: Optional[DeltaMessage]
401

402
                    # handle streaming deltas for tools with named tool_choice
403
                    if tool_choice_function_name:
404
                        delta_message = DeltaMessage(tool_calls=[
405
                            DeltaToolCall(function=DeltaFunctionCall(
406
                                name=tool_choice_function_name,
407
408
                                arguments=delta_text),
                                          index=i)
409
                        ])
410
411

                    # handle streaming deltas for tools with "auto" tool choice
412
413
414
415
416
417
418
419
420
421
422
                    elif tool_choice_auto:
                        assert previous_texts is not None
                        assert all_previous_token_ids is not None
                        assert tool_parser is not None
                        #TODO optimize manipulation of these lists
                        previous_text = previous_texts[i]
                        previous_token_ids = all_previous_token_ids[i]
                        current_text = previous_text + delta_text
                        current_token_ids = previous_token_ids + list(
                            output.token_ids)

423
424
                        delta_message = (
                            tool_parser.extract_tool_calls_streaming(
425
426
                                previous_text=previous_text,
                                current_text=current_text,
427
                                delta_text=delta_text,
428
429
                                previous_token_ids=previous_token_ids,
                                current_token_ids=current_token_ids,
430
431
                                delta_token_ids=output.token_ids,
                                request=request))
432
433
434
435

                        # update the previous values for the next iteration
                        previous_texts[i] = current_text
                        all_previous_token_ids[i] = current_token_ids
436
437

                    # handle streaming just a content delta
438
439
440
                    else:
                        delta_message = DeltaMessage(content=delta_text)

441
                    # set the previous values for the next iteration
442
                    previous_num_tokens[i] += len(output.token_ids)
443
444
445
446
447
448
449
450

                    # if the message delta is None (e.g. because it was a
                    # "control token" for tool calls or the parser otherwise
                    # wasn't ready to send a token, then
                    #   get the next token without streaming a chunk
                    if delta_message is None:
                        continue

451
452
453
454
                    if output.finish_reason is None:
                        # Send token-by-token response for each request.n
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
455
                            delta=delta_message,
456
457
                            logprobs=logprobs,
                            finish_reason=None)
458
459

                    # if the model is finished generating
460
                    else:
461
462
463
464
                        # check to make sure we haven't "forgotten" to stream
                        #   any tokens that were generated but previously
                        #   matched by partial json parsing
                        # only happens if we are NOT using guided decoding
465
                        auto_tools_called = False
466
                        if tool_parser:
467
468
469
470
                            auto_tools_called = len(
                                tool_parser.prev_tool_call_arr) > 0
                            index = len(tool_parser.prev_tool_call_arr
                                        ) - 1 if auto_tools_called else 0
471
472
473
474
475
476
477
478
479
480
481
                        else:
                            index = 0

                        if self._should_check_for_unstreamed_tool_arg_tokens(
                                delta_message, output) and tool_parser:
                            # get the expected call based on partial JSON
                            # parsing which "autocompletes" the JSON
                            expected_call = json.dumps(
                                tool_parser.prev_tool_call_arr[index].get(
                                    "arguments", {}))

482
                            # get what we've streamed so far for arguments
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
                            # for the current tool
                            actual_call = tool_parser.streamed_args_for_tool[
                                index]

                            # check to see if there's anything left to stream
                            remaining_call = expected_call.replace(
                                actual_call, "", 1)

                            # set that as a delta message
                            delta_message = DeltaMessage(tool_calls=[
                                DeltaToolCall(index=index,
                                              function=DeltaFunctionCall(
                                                  arguments=remaining_call).
                                              model_dump(exclude_none=True))
                            ])

499
500
501
                        # Send the finish response for each request.n only once
                        choice_data = ChatCompletionResponseStreamChoice(
                            index=i,
502
                            delta=delta_message,
503
                            logprobs=logprobs,
504
                            finish_reason=output.finish_reason
505
                            if not auto_tools_called else "tool_calls",
506
                            stop_reason=output.stop_reason)
507

508
                        finish_reason_sent[i] = True
509

510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
                    chunk = ChatCompletionStreamResponse(
                        id=request_id,
                        object=chunk_object_type,
                        created=created_time,
                        choices=[choice_data],
                        model=model_name)

                    # handle usage stats if requested & if continuous
                    if include_continuous_usage:
                        completion_tokens = previous_num_tokens[i]
                        chunk.usage = UsageInfo(
                            prompt_tokens=num_prompt_tokens,
                            completion_tokens=completion_tokens,
                            total_tokens=num_prompt_tokens + completion_tokens,
                        )

                    data = chunk.model_dump_json(exclude_unset=True)
                    yield f"data: {data}\n\n"

529
530
            # once the final token is handled, if stream_options.include_usage
            # is sent, send the usage
531
532
            if include_usage:
                completion_tokens = sum(previous_num_tokens)
533
                final_usage = UsageInfo(
534
535
536
                    prompt_tokens=num_prompt_tokens,
                    completion_tokens=completion_tokens,
                    total_tokens=num_prompt_tokens + completion_tokens,
537
538
539
540
541
542
543
544
545
546
547
548
                )

                final_usage_chunk = ChatCompletionStreamResponse(
                    id=request_id,
                    object=chunk_object_type,
                    created=created_time,
                    choices=[],
                    model=model_name,
                    usage=final_usage)
                final_usage_data = (final_usage_chunk.model_dump_json(
                    exclude_unset=True, exclude_none=True))
                yield f"data: {final_usage_data}\n\n"
549

550
551
552
553
554
555
556
            # report to FastAPI middleware aggregate usage across all choices
            num_completion_tokens = sum(previous_num_tokens)
            request_metadata.final_usage_info = UsageInfo(
                prompt_tokens=num_prompt_tokens,
                completion_tokens=num_completion_tokens,
                total_tokens=num_prompt_tokens + num_completion_tokens)

557
558
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
559
            logger.exception("Error in chat completion stream generator.")
560
561
            data = self.create_streaming_error_response(str(e))
            yield f"data: {data}\n\n"
562
563
564
565
        # Send the final done message after all response.n are finished
        yield "data: [DONE]\n\n"

    async def chat_completion_full_generator(
566
567
568
569
570
        self,
        request: ChatCompletionRequest,
        result_generator: AsyncIterator[RequestOutput],
        request_id: str,
        conversation: List[ConversationMessage],
571
        tokenizer: AnyTokenizer,
572
        request_metadata: RequestResponseMetadata,
573
    ) -> Union[ErrorResponse, ChatCompletionResponse]:
574

575
        model_name = self.base_model_paths[0].name
576
        created_time = int(time.time())
577
        final_res: Optional[RequestOutput] = None
578

579
580
581
582
583
        try:
            async for res in result_generator:
                final_res = res
        except asyncio.CancelledError:
            return self.create_error_response("Client disconnected")
584
585
586
        except ValueError as e:
            # TODO: Use a vllm-specific Validation Error
            return self.create_error_response(str(e))
587

588
589
        assert final_res is not None

590
        choices: List[ChatCompletionResponseChoice] = []
591

592
593
        role = self.get_chat_request_role(request)
        for output in final_res.outputs:
594
            token_ids = output.token_ids
595
            out_logprobs = output.logprobs
596

597
598
            if request.logprobs and request.top_logprobs is not None:
                assert out_logprobs is not None, "Did not output logprobs"
599
                logprobs = self._create_chat_logprobs(
600
                    token_ids=token_ids,
601
                    top_logprobs=out_logprobs,
602
                    num_output_top_logprobs=request.top_logprobs,
603
                    tokenizer=tokenizer,
604
605
606
607
                )
            else:
                logprobs = None

608
609
610
611
            # In the OpenAI API the finish_reason is "tools_called"
            # if the tool choice is auto and the model produced a tool
            # call. The same is not true for named function calls
            auto_tools_called = False
612
613
614

            # if auto tools are not enabled, and a named tool choice using
            #   outlines is not being used
615
            if (not self.enable_auto_tools
616
617
618
619
620
621
622
                    or not self.tool_parser) and not isinstance(
                        request.tool_choice,
                        ChatCompletionNamedToolChoiceParam):
                message = ChatMessage(role=role, content=output.text)

            # if the request uses tools and specified a tool choice
            elif request.tool_choice and type(
623
                    request.tool_choice) is ChatCompletionNamedToolChoiceParam:
624

625
626
627
628
629
630
631
632
                message = ChatMessage(
                    role=role,
                    content="",
                    tool_calls=[
                        ToolCall(function=FunctionCall(
                            name=request.tool_choice.function.name,
                            arguments=output.text))
                    ])
633
634
635

            # if the request doesn't use tool choice
            # OR specifies to not use a tool
636
            elif not request.tool_choice or request.tool_choice == "none":
637
638
639
640
641
642
643
644
645

                message = ChatMessage(role=role, content=output.text)

            # handle when there are tools and tool choice is auto
            elif request.tools and (
                    request.tool_choice == "auto"
                    or request.tool_choice is None) and self.enable_auto_tools \
                    and self.tool_parser:

646
647
648
                try:
                    tool_parser = self.tool_parser(tokenizer)
                except RuntimeError as e:
649
                    logger.exception("Error in tool parser creation.")
650
651
                    return self.create_error_response(str(e))

652
653
                tool_call_info = tool_parser.extract_tool_calls(
                    output.text, request=request)
654
655
656
657
                # In the OpenAI API the finish_reason is "tools_called"
                # if the tool choice is auto and the model produced a tool
                # call. The same is not true for named function calls
                auto_tools_called = tool_call_info.tools_called
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
                if tool_call_info.tools_called:
                    message = ChatMessage(role=role,
                                          content=tool_call_info.content,
                                          tool_calls=tool_call_info.tool_calls)

                else:
                    # FOR NOW make it a chat message; we will have to detect
                    # the type to make it later.
                    message = ChatMessage(role=role, content=output.text)

            # undetermined case that is still important to handle
            else:
                logger.error(
                    "Error in chat_completion_full_generator - cannot determine"
                    " if tools should be extracted. Returning a standard chat "
                    "completion.")
674
675
                message = ChatMessage(role=role, content=output.text)

676
677
            choice_data = ChatCompletionResponseChoice(
                index=output.index,
678
                message=message,
679
                logprobs=logprobs,
680
                finish_reason="tool_calls" if auto_tools_called else
681
                output.finish_reason if output.finish_reason else "stop",
682
                stop_reason=output.stop_reason)
683
684
            choices.append(choice_data)

685
        if request.echo or request.continue_final_message:
686
            last_msg_content: Union[str, List[Dict[str, str]]] = ""
687
688
            if conversation and "content" in conversation[-1] and conversation[
                    -1].get("role") == role:
689
                last_msg_content = conversation[-1]["content"] or ""
690
691
692
            if isinstance(last_msg_content, list):
                last_msg_content = "\n".join(msg['text']
                                             for msg in last_msg_content)
693
694

            for choice in choices:
695
696
                full_message = last_msg_content + (choice.message.content
                                                   or "")
697
698
                choice.message.content = full_message

699
        assert final_res.prompt_token_ids is not None
700
        num_prompt_tokens = len(final_res.prompt_token_ids)
701
702
        if final_res.encoder_prompt_token_ids is not None:
            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
703
704
705
706
707
708
709
        num_generated_tokens = sum(
            len(output.token_ids) for output in final_res.outputs)
        usage = UsageInfo(
            prompt_tokens=num_prompt_tokens,
            completion_tokens=num_generated_tokens,
            total_tokens=num_prompt_tokens + num_generated_tokens,
        )
710
711
712

        request_metadata.final_usage_info = usage

713
714
715
716
717
718
        response = ChatCompletionResponse(
            id=request_id,
            created=created_time,
            model=model_name,
            choices=choices,
            usage=usage,
719
            prompt_logprobs=final_res.prompt_logprobs,
720
721
        )

722
        return response
723
724

    def _get_top_logprobs(
725
            self, logprobs: Dict[int, Logprob], top_logprobs: Optional[int],
726
            tokenizer: AnyTokenizer) -> List[ChatCompletionLogProb]:
727
        return [
728
729
730
731
732
733
734
735
            ChatCompletionLogProb(token=(token := self._get_decoded_token(
                p[1],
                p[0],
                tokenizer,
                return_as_token_id=self.return_tokens_as_token_ids)),
                                  logprob=max(p[1].logprob, -9999.0),
                                  bytes=list(
                                      token.encode("utf-8", errors="replace")))
736
737
738
739
740
741
742
743
            for i, p in enumerate(logprobs.items())
            if top_logprobs and i < top_logprobs
        ]

    def _create_chat_logprobs(
        self,
        token_ids: GenericSequence[int],
        top_logprobs: GenericSequence[Optional[Dict[int, Logprob]]],
744
        tokenizer: AnyTokenizer,
745
746
747
        num_output_top_logprobs: Optional[int] = None,
    ) -> ChatCompletionLogProbs:
        """Create OpenAI-style logprobs."""
748
        logprobs_content: List[ChatCompletionLogProbsContent] = []
749
750
751
752

        for i, token_id in enumerate(token_ids):
            step_top_logprobs = top_logprobs[i]
            if step_top_logprobs is None:
753
                token = tokenizer.decode(token_id)
754
755
                if self.return_tokens_as_token_ids:
                    token = f"token_id:{token_id}"
756

757
758
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
759
                        token=token,
760
761
                        bytes=list(token.encode("utf-8", errors="replace")),
                    ))
762
            else:
763
764
765
                step_token = step_top_logprobs[token_id]
                step_decoded = step_token.decoded_token

766
767
                logprobs_content.append(
                    ChatCompletionLogProbsContent(
768
                        token=self._get_decoded_token(
769
770
771
772
773
774
775
776
                            step_token,
                            token_id,
                            tokenizer,
                            self.return_tokens_as_token_ids,
                        ),
                        logprob=max(step_token.logprob, -9999.0),
                        bytes=None if step_decoded is None else list(
                            step_decoded.encode("utf-8", errors="replace")),
777
                        top_logprobs=self._get_top_logprobs(
778
779
780
781
782
                            step_top_logprobs,
                            num_output_top_logprobs,
                            tokenizer,
                        ),
                    ))
783
784

        return ChatCompletionLogProbs(content=logprobs_content)
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813

    def _should_stream_with_auto_tool_parsing(self,
                                              request: ChatCompletionRequest):
        """
        Utility function to check if streamed tokens should go through the tool
        call parser that was configured.

        We only want to do this IF user-provided tools are set, a tool parser
        is configured, "auto" tool choice is enabled, and the request's tool
        choice field indicates that "auto" tool choice should be used.
        """
        return (request.tools and self.tool_parser and self.enable_auto_tools
                and request.tool_choice in ['auto', None])

    def _should_check_for_unstreamed_tool_arg_tokens(
        self,
        delta_message: Optional[DeltaMessage],
        output: CompletionOutput,
    ) -> bool:
        """
        Check to see if we should check for unstreamed tool arguments tokens.
        This is only applicable when auto tool parsing is enabled, the delta
        is a tool call with arguments.
        """

        # yapf: disable
        return bool(
            # if there is a delta message that includes tool calls which
            # include a function that has arguments
814
815
            output.finish_reason is not None
            and self.enable_auto_tools and self.tool_parser and delta_message
816
817
818
819
            and delta_message.tool_calls and delta_message.tool_calls[0]
            and delta_message.tool_calls[0].function
            and delta_message.tool_calls[0].function.arguments is not None
        )