chat_utils.py 46.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import json
6
from abc import ABC, abstractmethod
7
from collections import defaultdict, deque
8
from collections.abc import Awaitable, Iterable
9
from functools import cache, lru_cache, partial
10
from pathlib import Path
11
12
from typing import (Any, Callable, Generic, Literal, Optional, TypeVar, Union,
                    cast)
13

14
15
import jinja2.nodes
import transformers.utils.chat_template_utils as hf_chat_utils
16
17
# yapf conflicts with isort for this block
# yapf: disable
18
from openai.types.chat import (ChatCompletionAssistantMessageParam,
19
20
                               ChatCompletionContentPartImageParam,
                               ChatCompletionContentPartInputAudioParam)
21
22
from openai.types.chat import (
    ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam)
23
24
from openai.types.chat import (ChatCompletionContentPartRefusalParam,
                               ChatCompletionContentPartTextParam)
25
26
from openai.types.chat import (
    ChatCompletionMessageParam as OpenAIChatCompletionMessageParam)
27
28
from openai.types.chat import (ChatCompletionMessageToolCallParam,
                               ChatCompletionToolMessageParam)
29
30
from openai.types.chat.chat_completion_content_part_input_audio_param import (
    InputAudio)
31
from pydantic import TypeAdapter
32
# yapf: enable
33
34
from transformers import (PreTrainedTokenizer, PreTrainedTokenizerFast,
                          ProcessorMixin)
35
# pydantic needs the TypedDict from typing_extensions
36
from typing_extensions import Required, TypeAlias, TypedDict
37

38
from vllm.config import ModelConfig
39
from vllm.logger import init_logger
40
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
41
from vllm.multimodal.utils import MediaConnector
42
43
44
45
# yapf: disable
from vllm.transformers_utils.chat_templates import (
    get_chat_template_fallback_path)
# yapf: enable
46
from vllm.transformers_utils.processor import cached_get_processor
47
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
48
from vllm.utils import deprecate_kwargs, random_uuid
49
50
51
52

logger = init_logger(__name__)


53
54
55
56
57
58
59
60
61
62
63
64
65
66
class AudioURL(TypedDict, total=False):
    url: Required[str]
    """
    Either a URL of the audio or a data URL with base64 encoded audio data.
    """


class ChatCompletionContentPartAudioParam(TypedDict, total=False):
    audio_url: Required[AudioURL]

    type: Required[Literal["audio_url"]]
    """The type of the content part."""


67
68
69
70
71
72
73
74
75
76
77
class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
    image_embeds: Required[Union[str, dict[str, str]]]
    """
    The image embeddings. It can be either:
    - A single base64 string.
    - A dictionary where each value is a base64 string.
    """
    type: Required[Literal["image_embeds"]]
    """The type of the content part."""


78
79
80
81
82
83
84
85
86
87
88
89
90
91
class VideoURL(TypedDict, total=False):
    url: Required[str]
    """
    Either a URL of the video or a data URL with base64 encoded video data.
    """


class ChatCompletionContentPartVideoParam(TypedDict, total=False):
    video_url: Required[VideoURL]

    type: Required[Literal["video_url"]]
    """The type of the content part."""


92
93
94
class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain image_url.
    This is supported by OpenAI API, although it is not documented.
95

96
97
98
99
100
101
102
103
104
105
    Example:
    {
        "image_url": "https://example.com/image.jpg"
    }
    """
    image_url: Required[str]


class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain audio_url.
106

107
108
109
110
111
112
113
114
    Example:
    {
        "audio_url": "https://example.com/audio.mp3"
    }
    """
    audio_url: Required[str]


115
116
117
118
119
120
121
122
123
124
125
class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain audio_url.

    Example:
    {
        "video_url": "https://example.com/video.mp4"
    }
    """
    video_url: Required[str]


126
127
ChatCompletionContentPartParam: TypeAlias = Union[
    OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
128
    ChatCompletionContentPartInputAudioParam,
129
    ChatCompletionContentPartVideoParam, ChatCompletionContentPartRefusalParam,
130
    CustomChatCompletionContentSimpleImageParam,
131
    ChatCompletionContentPartImageEmbedsParam,
132
133
    CustomChatCompletionContentSimpleAudioParam,
    CustomChatCompletionContentSimpleVideoParam, str]
134
135
136
137
138
139
140


class CustomChatCompletionMessageParam(TypedDict, total=False):
    """Enables custom roles in the Chat Completion API."""
    role: Required[str]
    """The role of the message's author."""

141
    content: Union[str, list[ChatCompletionContentPartParam]]
142
143
144
145
146
147
148
149
150
    """The contents of the message."""

    name: str
    """An optional name for the participant.

    Provides the model information to differentiate between participants of the
    same role.
    """

151
152
153
154
155
156
    tool_call_id: Optional[str]
    """Tool call that this message is responding to."""

    tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
    """The tool calls generated by the model, such as function calls."""

157
158
159
160
161

ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam,
                                   CustomChatCompletionMessageParam]


162
# TODO: Make fields ReadOnly once mypy supports it
163
164
165
166
class ConversationMessage(TypedDict, total=False):
    role: Required[str]
    """The role of the message's author."""

167
    content: Union[Optional[str], list[dict[str, str]]]
168
169
170
171
172
173
174
175
176
177
    """The contents of the message"""

    tool_call_id: Optional[str]
    """Tool call that this message is responding to."""

    name: Optional[str]
    """The name of the function to call"""

    tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
    """The tool calls generated by the model, such as function calls."""
178
179


180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
# Passed in by user
ChatTemplateContentFormatOption = Literal["auto", "string", "openai"]

# Used internally
_ChatTemplateContentFormat = Literal["string", "openai"]


def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool:
    if isinstance(node, jinja2.nodes.Name):
        return node.ctx == "load" and node.name == varname

    return False


def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
    if isinstance(node, jinja2.nodes.Getitem):
        return (_is_var_access(node.node, varname)
                and isinstance(node.arg, jinja2.nodes.Const)
                and node.arg.value == key)

    if isinstance(node, jinja2.nodes.Getattr):
        return _is_var_access(node.node, varname) and node.attr == key

    return False


def _is_var_or_elems_access(
    node: jinja2.nodes.Node,
    varname: str,
    key: Optional[str] = None,
) -> bool:
    if isinstance(node, jinja2.nodes.Filter):
        return (node.node is not None
                and _is_var_or_elems_access(node.node, varname, key))
    if isinstance(node, jinja2.nodes.Test):
        return _is_var_or_elems_access(node.node, varname, key)

    if (isinstance(node, jinja2.nodes.Getitem)
            and isinstance(node.arg, jinja2.nodes.Slice)):
        return _is_var_or_elems_access(node.node, varname, key)

    # yapf: disable
    return (
        _is_attr_access(node, varname, key) if key
        else _is_var_access(node, varname)
    ) # yapf: enable


def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str):
    # Global variable that is implicitly defined at the root
    yield root, varname

    # Iterative BFS
    related_varnames = deque([varname])
    while related_varnames:
        related_varname = related_varnames.popleft()

        for assign_ast in root.find_all(jinja2.nodes.Assign):
            lhs = assign_ast.target
            rhs = assign_ast.node

            if _is_var_or_elems_access(rhs, related_varname):
                assert isinstance(lhs, jinja2.nodes.Name)
                yield assign_ast, lhs.name

                # Avoid infinite looping for self-assignment
                if lhs.name != related_varname:
                    related_varnames.append(lhs.name)


# NOTE: The proper way to handle this is to build a CFG so that we can handle
# the scope in which each variable is defined, but that is too complicated
def _iter_nodes_assign_messages_item(root: jinja2.nodes.Node):
    messages_varnames = [
        varname
        for _, varname in _iter_nodes_assign_var_or_elems(root, "messages")
    ]

    # Search for {%- for message in messages -%} loops
    for loop_ast in root.find_all(jinja2.nodes.For):
        loop_iter = loop_ast.iter
        loop_target = loop_ast.target

        for varname in messages_varnames:
            if _is_var_or_elems_access(loop_iter, varname):
                assert isinstance(loop_target, jinja2.nodes.Name)
                yield loop_ast, loop_target.name
                break


def _iter_nodes_assign_content_item(root: jinja2.nodes.Node):
    message_varnames = [
        varname for _, varname in _iter_nodes_assign_messages_item(root)
    ]

    # Search for {%- for content in message['content'] -%} loops
    for loop_ast in root.find_all(jinja2.nodes.For):
        loop_iter = loop_ast.iter
        loop_target = loop_ast.target

        for varname in message_varnames:
            if _is_var_or_elems_access(loop_iter, varname, "content"):
                assert isinstance(loop_target, jinja2.nodes.Name)
                yield loop_ast, loop_target.name
                break


def _try_extract_ast(chat_template: str) -> Optional[jinja2.nodes.Template]:
    try:
        jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
        return jinja_compiled.environment.parse(chat_template)
    except Exception:
        logger.exception("Error when compiling Jinja template")
        return None


296
@lru_cache(maxsize=32)
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
def _detect_content_format(
    chat_template: str,
    *,
    default: _ChatTemplateContentFormat,
) -> _ChatTemplateContentFormat:
    jinja_ast = _try_extract_ast(chat_template)
    if jinja_ast is None:
        return default

    try:
        next(_iter_nodes_assign_content_item(jinja_ast))
    except StopIteration:
        return "string"
    except Exception:
        logger.exception("Error when parsing AST of Jinja template")
        return default
    else:
        return "openai"


317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
def resolve_mistral_chat_template(
    chat_template: Optional[str],
    **kwargs: Any,
) -> Optional[str]:
    if chat_template is not None:
        logger.warning_once(
            "'chat_template' cannot be overridden for mistral tokenizer.")
    if "add_generation_prompt" in kwargs:
        logger.warning_once(
            "'add_generation_prompt' is not supported for mistral tokenizer, "
            "so it will be ignored.")
    if "continue_final_message" in kwargs:
        logger.warning_once(
            "'continue_final_message' is not supported for mistral tokenizer, "
            "so it will be ignored.")
    return None

334
335
336
337
@deprecate_kwargs(
    "trust_remote_code",
    additional_message="Please use `model_config.trust_remote_code` instead.",
)
338
def resolve_hf_chat_template(
339
340
341
    tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
    chat_template: Optional[str],
    tools: Optional[list[dict[str, Any]]],
342
343
    *,
    model_config: ModelConfig,
344
    trust_remote_code: Optional[bool] = None,
345
346
347
348
349
350
351
352
353
354
355
356
) -> Optional[str]:
    # 1st priority: The given chat template
    if chat_template is not None:
        return chat_template

    # 2nd priority: AutoProcessor chat template, unless tool calling is enabled
    if tools is None:
        try:
            processor = cached_get_processor(
                tokenizer.name_or_path,
                processor_cls=(PreTrainedTokenizer, PreTrainedTokenizerFast,
                               ProcessorMixin),
357
                trust_remote_code=model_config.trust_remote_code,
358
359
            )
            if isinstance(processor, ProcessorMixin) and \
360
                hasattr(processor, 'chat_template') and \
361
362
363
                processor.chat_template is not None:
                return processor.chat_template
        except Exception:
364
            logger.debug("Failed to load AutoProcessor chat template for %s", tokenizer.name_or_path, exc_info=True)  # noqa: E501
365
366
367
368
369
370
371
372

    # 3rd priority: AutoTokenizer chat template
    try:
        return tokenizer.get_chat_template(chat_template, tools=tools)
    except Exception:
        logger.debug("Failed to load AutoTokenizer chat template for %s",
                     tokenizer.name_or_path, exc_info=True)

373
374
375
376
377
378
379
380
381
382
383
384
385
386
    # 4th priority: Predefined fallbacks
    path = get_chat_template_fallback_path(
        model_type=model_config.hf_config.model_type,
        tokenizer_name_or_path=model_config.tokenizer,
    )
    if path is not None:
        logger.info("Loading chat template fallback for %s as there isn't one "
                    "defined on HF Hub.", tokenizer.name_or_path)
        chat_template = load_chat_template(path)
    else:
        logger.debug("There is no chat template fallback for %s",
                     tokenizer.name_or_path)

    return chat_template
387
388


389
390
def _resolve_chat_template_content_format(
    chat_template: Optional[str],
391
    tools: Optional[list[dict[str, Any]]],
392
    tokenizer: AnyTokenizer,
393
394
    *,
    model_config: ModelConfig,
395
396
) -> _ChatTemplateContentFormat:
    if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
397
        hf_chat_template = resolve_hf_chat_template(
398
399
400
            tokenizer,
            chat_template=chat_template,
            tools=tools,
401
            model_config=model_config,
402
        )
403
    else:
404
405
406
407
        hf_chat_template = None

    jinja_text = (hf_chat_template if isinstance(hf_chat_template, str)
                  else load_chat_template(chat_template, is_literal=True))
408
409
410
411

    detected_format = ("string" if jinja_text is None else
                       _detect_content_format(jinja_text, default="string"))

412
    return detected_format
413
414
415


@lru_cache
416
def _log_chat_template_content_format(
417
418
    chat_template: Optional[str],
    given_format: ChatTemplateContentFormatOption,
419
420
    detected_format: ChatTemplateContentFormatOption,
):
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
    logger.info(
        "Detected the chat template content format to be '%s'. "
        "You can set `--chat-template-content-format` to override this.",
        detected_format,
    )

    if given_format != "auto" and given_format != detected_format:
        logger.warning(
            "You specified `--chat-template-content-format %s` "
            "which is different from the detected format '%s'. "
            "If our automatic detection is incorrect, please consider "
            "opening a GitHub issue so that we can improve it: "
            "https://github.com/vllm-project/vllm/issues/new/choose",
            given_format,
            detected_format,
        )

438

439
440
441
442
@deprecate_kwargs(
    "trust_remote_code",
    additional_message="Please use `model_config.trust_remote_code` instead.",
)
443
444
445
446
447
def resolve_chat_template_content_format(
    chat_template: Optional[str],
    tools: Optional[list[dict[str, Any]]],
    given_format: ChatTemplateContentFormatOption,
    tokenizer: AnyTokenizer,
448
449
450
    *,
    model_config: ModelConfig,
    trust_remote_code: Optional[bool] = None,
451
) -> _ChatTemplateContentFormat:
452
453
454
    if given_format != "auto":
        return given_format

455
456
457
458
    detected_format = _resolve_chat_template_content_format(
        chat_template,
        tools,
        tokenizer,
459
        model_config=model_config,
460
461
462
463
464
465
466
467
    )

    _log_chat_template_content_format(
        chat_template,
        given_format=given_format,
        detected_format=detected_format,
    )

468
    return detected_format
469

470
471


472
ModalityStr = Literal["image", "audio", "video", "image_embeds"]
473
474
475
476
_T = TypeVar("_T")


class BaseMultiModalItemTracker(ABC, Generic[_T]):
477
478
479
480
481
482
483
    """
    Tracks multi-modal items in a given request and ensures that the number
    of multi-modal items in a given request does not exceed the configured
    maximum per prompt.
    """

    def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer):
484
485
        super().__init__()

486
487
        self._model_config = model_config
        self._tokenizer = tokenizer
488

489
        self._items_by_modality = defaultdict[str, list[_T]](list)
490

491
492
493
494
    @property
    def model_config(self) -> ModelConfig:
        return self._model_config

495
496
497
498
    @property
    def allowed_local_media_path(self):
        return self._model_config.allowed_local_media_path

499
500
501
502
    @property
    def mm_registry(self):
        return MULTIMODAL_REGISTRY

503
    @staticmethod
504
    @cache
505
    def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
506
507
        return tokenizer.decode(token_index)

508
509
    def _placeholder_str(self, modality: ModalityStr,
                         current_count: int) -> Optional[str]:
510
511
        # TODO: Let user specify how to insert image tokens into prompt
        # (similar to chat template)
512
513
514
        hf_config = self._model_config.hf_config
        model_type = hf_config.model_type

515
516
517
        if modality in ("image", "image_embeds"):
            if model_type == "chatglm":
                return "<|begin_of_image|><|endoftext|><|end_of_image|>"
518
519
            if model_type == "glm4v":
                return "<|begin_of_image|><|image|><|end_of_image|>"
520
            if model_type in ("phi3_v", "phi4mm"):
521
                return f"<|image_{current_count}|>"
522
            if model_type in ("minicpmo", "minicpmv"):
523
                return "(<image>./</image>)"
524
525
            if model_type in ("blip-2", "florence2", "fuyu", "paligemma",
                              "pixtral", "mistral3"):
526
527
                # These models do not use image tokens in the prompt
                return None
528
529
            if model_type == "qwen":
                return f"Picture {current_count}: <img></img>"
530
            if model_type.startswith("llava"):
531
532
                return self._cached_token_str(self._tokenizer,
                                              hf_config.image_token_index)
533

Jennifer Zhao's avatar
Jennifer Zhao committed
534
            if model_type in ("aya_vision", "chameleon", "deepseek_vl_v2",
535
                              "internvl_chat", "ovis", "skywork_chat",
536
                              "NVLM_D", "h2ovl_chat", "idefics3", "smolvlm"):
537
                return "<image>"
538
            if model_type in ("mllama", "llama4"):
539
                return "<|image|>"
Roger Wang's avatar
Roger Wang committed
540
            if model_type in ("qwen2_vl", "qwen2_5_vl"):
541
                return "<|vision_start|><|image_pad|><|vision_end|>"
542
543
            if model_type == "qwen2_5_omni":
                return "<|vision_start|><|IMAGE|><|vision_end|>"
544
545
            if model_type == "molmo":
                return ""
546
547
            if model_type == "aria":
                return "<|fim_prefix|><|img|><|fim_suffix|>"
548
549
            if model_type == "gemma3":
                return "<start_of_image>"
550
551
            if model_type == "kimi_vl":
                return "<|media_start|>image<|media_content|><|media_pad|><|media_end|>" # noqa: E501
552

553
            raise TypeError(f"Unknown {modality} model type: {model_type}")
554
        elif modality == "audio":
555
            if model_type in ("ultravox", "granite_speech"):
556
                return "<|audio|>"
557
            if model_type == "phi4mm":
558
                return f"<|audio_{current_count}|>"
559
            if model_type in ("qwen2_audio", "qwen2_5_omni"):
560
561
                return (f"Audio {current_count}: "
                        f"<|audio_bos|><|AUDIO|><|audio_eos|>")
562
563
            if model_type == "minicpmo":
                return "(<audio>./</audio>)"
564
            raise TypeError(f"Unknown model type: {model_type}")
565
        elif modality == "video":
566
567
            if model_type == "internvl_chat":
                return "<video>"
568
569
            if model_type == "glm4v":
                return "<|begin_of_video|><|video|><|end_of_video|>"
Roger Wang's avatar
Roger Wang committed
570
            if model_type in ("qwen2_vl", "qwen2_5_vl"):
571
                return "<|vision_start|><|video_pad|><|vision_end|>"
572
573
            if model_type == "qwen2_5_omni":
                return "<|vision_start|><|VIDEO|><|vision_end|>"
574
575
            if model_type in ("minicpmo", "minicpmv"):
                return "(<video>./</video>)"
576
577
578
            if model_type.startswith("llava"):
                return self._cached_token_str(self._tokenizer,
                                              hf_config.video_token_index)
579
            raise TypeError(f"Unknown {modality} model type: {model_type}")
580
581
582
        else:
            raise TypeError(f"Unknown modality: {modality}")

583
584
585
586
587
    def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
        """
        Add a multi-modal item to the current prompt and returns the
        placeholder string to use, if any.
        """
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
        mm_registry = self.mm_registry
        model_config = self.model_config

        input_modality = modality.replace("_embeds", "")

        if mm_registry.has_processor(model_config):
            mm_processor = mm_registry.create_processor(model_config)
            allowed_counts = mm_processor.info.get_allowed_mm_limits()
            allowed_count = allowed_counts.get(input_modality, 0)
        else:
            mm_config = model_config.multimodal_config
            if mm_config is None:
                msg = "This model does not support multi-modal inputs"
                raise ValueError(msg)

            allowed_count = mm_config.get_limit_per_prompt(input_modality)

605
        current_count = len(self._items_by_modality[modality]) + 1
606
607
608
        if current_count > allowed_count:
            raise ValueError(
                f"At most {allowed_count} {modality}(s) may be provided in "
609
610
                "one request. You can set `--limit-mm-per-prompt` to "
                "increase this limit if the model supports it.")
611

612
        self._items_by_modality[modality].append(item)
613
614
615
616
617
618
619
620

        return self._placeholder_str(modality, current_count)

    @abstractmethod
    def create_parser(self) -> "BaseMultiModalContentParser":
        raise NotImplementedError


621
class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
622
623

    def all_mm_data(self) -> Optional[MultiModalDataDict]:
624
625
626
627
628
629
630
631
632
633
634
635
636
637
        if not self._items_by_modality:
            return None
        mm_inputs = {}
        items_by_modality = dict(self._items_by_modality)
        if "image" in items_by_modality and "image_embeds" in items_by_modality:
            raise ValueError(\
                "Mixing raw image and embedding inputs is not allowed")

        if "image_embeds" in items_by_modality:
            image_embeds_lst = items_by_modality["image_embeds"]
            if len(image_embeds_lst) > 1:
                raise ValueError(\
                    "Only one message can have {'type': 'image_embeds'}")
            mm_inputs["image"] = image_embeds_lst[0]
638
        if "image" in items_by_modality:
639
            mm_inputs["image"] = items_by_modality["image"] # A list of images
640
        if "audio" in items_by_modality:
641
            mm_inputs["audio"] = items_by_modality["audio"] # A list of audios
642
        if "video" in items_by_modality:
643
644
            mm_inputs["video"] = items_by_modality["video"] # A list of videos
        return mm_inputs
645
646
647
648
649

    def create_parser(self) -> "BaseMultiModalContentParser":
        return MultiModalContentParser(self)


650
class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
651
652

    async def all_mm_data(self) -> Optional[MultiModalDataDict]:
653
654
655
656
        if not self._items_by_modality:
            return None
        mm_inputs = {}
        items_by_modality = {
657
658
659
                modality: await asyncio.gather(*items)
                for modality, items in self._items_by_modality.items()
            }
660

661
662
663
664
665
666
667
668
669
670
        if "image" in items_by_modality and "image_embeds" in items_by_modality:
            raise ValueError(
                "Mixing raw image and embedding inputs is not allowed")

        if "image_embeds" in items_by_modality:
            image_embeds_lst = items_by_modality["image_embeds"]
            if len(image_embeds_lst) > 1:
                raise ValueError(
                    "Only one message can have {'type': 'image_embeds'}")
            mm_inputs["image"] = image_embeds_lst[0]
671
        if "image" in items_by_modality:
672
            mm_inputs["image"] = items_by_modality["image"] # A list of images
673
        if "audio" in items_by_modality:
674
            mm_inputs["audio"] = items_by_modality["audio"] # A list of audios
675
        if "video" in items_by_modality:
676
677
            mm_inputs["video"] = items_by_modality["video"] # A list of videos
        return mm_inputs
678
679
680
681
682
683
684
685
686
687
688

    def create_parser(self) -> "BaseMultiModalContentParser":
        return AsyncMultiModalContentParser(self)


class BaseMultiModalContentParser(ABC):

    def __init__(self) -> None:
        super().__init__()

        # multimodal placeholder_string : count
689
        self._placeholder_counts: dict[str, int] = defaultdict(lambda: 0)
690
691
692
693
694

    def _add_placeholder(self, placeholder: Optional[str]):
        if placeholder:
            self._placeholder_counts[placeholder] += 1

695
    def mm_placeholder_counts(self) -> dict[str, int]:
696
697
698
699
700
701
        return dict(self._placeholder_counts)

    @abstractmethod
    def parse_image(self, image_url: str) -> None:
        raise NotImplementedError

702
703
704
705
706
    @abstractmethod
    def parse_image_embeds(self,
                           image_embeds: Union[str, dict[str, str]]) -> None:
        raise NotImplementedError

707
708
709
710
    @abstractmethod
    def parse_audio(self, audio_url: str) -> None:
        raise NotImplementedError

711
    @abstractmethod
712
    def parse_input_audio(self, input_audio: InputAudio) -> None:
713
714
        raise NotImplementedError

715
716
717
718
    @abstractmethod
    def parse_video(self, video_url: str) -> None:
        raise NotImplementedError

719
720
721
722
723
724
725
726

class MultiModalContentParser(BaseMultiModalContentParser):

    def __init__(self, tracker: MultiModalItemTracker) -> None:
        super().__init__()

        self._tracker = tracker

727
728
729
730
        self._connector = MediaConnector(
            allowed_local_media_path=tracker.allowed_local_media_path,
        )

731
    def parse_image(self, image_url: str) -> None:
732
        image = self._connector.fetch_image(image_url)
733
734
735
736

        placeholder = self._tracker.add("image", image)
        self._add_placeholder(placeholder)

737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
    def parse_image_embeds(self,
                           image_embeds: Union[str, dict[str, str]]) -> None:
        if isinstance(image_embeds, dict):
            embeds = {
                k: self._connector.fetch_image_embedding(v)
                for k, v in image_embeds.items()
            }
            placeholder = self._tracker.add("image_embeds", embeds)

        if isinstance(image_embeds, str):
            embedding = self._connector.fetch_image_embedding(image_embeds)
            placeholder = self._tracker.add("image_embeds", embedding)

        self._add_placeholder(placeholder)

752
    def parse_audio(self, audio_url: str) -> None:
753
        audio = self._connector.fetch_audio(audio_url)
754
755
756
757

        placeholder = self._tracker.add("audio", audio)
        self._add_placeholder(placeholder)

758
759
760
761
    def parse_input_audio(self, input_audio: InputAudio) -> None:
        audio_data = input_audio.get("data", "")
        audio_format = input_audio.get("format", "")
        audio_url = f"data:audio/{audio_format};base64,{audio_data}"
762

763
        return self.parse_audio(audio_url)
764

765
    def parse_video(self, video_url: str) -> None:
766
        video = self._connector.fetch_video(video_url)
767
768
769
770

        placeholder = self._tracker.add("video", video)
        self._add_placeholder(placeholder)

771
772
773
774
775
776
777

class AsyncMultiModalContentParser(BaseMultiModalContentParser):

    def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:
        super().__init__()

        self._tracker = tracker
778
779
780
        self._connector = MediaConnector(
            allowed_local_media_path=tracker.allowed_local_media_path,
        )
781
782

    def parse_image(self, image_url: str) -> None:
783
        image_coro = self._connector.fetch_image_async(image_url)
784
785
786
787

        placeholder = self._tracker.add("image", image_coro)
        self._add_placeholder(placeholder)

788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
    def parse_image_embeds(self,
                           image_embeds: Union[str, dict[str, str]]) -> None:
        future: asyncio.Future[Union[str, dict[str, str]]] = asyncio.Future()

        if isinstance(image_embeds, dict):
            embeds = {
                k: self._connector.fetch_image_embedding(v)
                for k, v in image_embeds.items()
            }
            future.set_result(embeds)

        if isinstance(image_embeds, str):
            embedding = self._connector.\
                fetch_image_embedding(image_embeds)
            future.set_result(embedding)

        placeholder = self._tracker.add("image_embeds", future)
        self._add_placeholder(placeholder)

807
    def parse_audio(self, audio_url: str) -> None:
808
        audio_coro = self._connector.fetch_audio_async(audio_url)
809
810
811

        placeholder = self._tracker.add("audio", audio_coro)
        self._add_placeholder(placeholder)
812

813
814
815
816
    def parse_input_audio(self, input_audio: InputAudio) -> None:
        audio_data = input_audio.get("data", "")
        audio_format = input_audio.get("format", "")
        audio_url = f"data:audio/{audio_format};base64,{audio_data}"
817

818
        return self.parse_audio(audio_url)
819

820
    def parse_video(self, video_url: str) -> None:
821
        video = self._connector.fetch_video_async(video_url)
822
823
824
825

        placeholder = self._tracker.add("video", video)
        self._add_placeholder(placeholder)

826

827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
def validate_chat_template(chat_template: Optional[Union[Path, str]]):
    """Raises if the provided chat template appears invalid."""
    if chat_template is None:
        return

    elif isinstance(chat_template, Path) and not chat_template.exists():
        raise FileNotFoundError(
            "the supplied chat template path doesn't exist")

    elif isinstance(chat_template, str):
        JINJA_CHARS = "{}\n"
        if not any(c in chat_template
                   for c in JINJA_CHARS) and not Path(chat_template).exists():
            raise ValueError(
                f"The supplied chat template string ({chat_template}) "
                f"appears path-like, but doesn't exist!")

    else:
        raise TypeError(
            f"{type(chat_template)} is not a valid chat template type")


849
def _load_chat_template(
850
851
852
853
    chat_template: Optional[Union[Path, str]],
    *,
    is_literal: bool = False,
) -> Optional[str]:
854
855
    if chat_template is None:
        return None
856
857
858
859
860
861

    if is_literal:
        if isinstance(chat_template, Path):
            raise TypeError("chat_template is expected to be read directly "
                            "from its value")

862
        return chat_template
863

864
    try:
865
        with open(chat_template) as f:
866
            return f.read()
867
    except OSError as e:
868
869
870
        if isinstance(chat_template, Path):
            raise

871
872
873
874
875
876
        JINJA_CHARS = "{}\n"
        if not any(c in chat_template for c in JINJA_CHARS):
            msg = (f"The supplied chat template ({chat_template}) "
                   f"looks like a file path, but it failed to be "
                   f"opened. Reason: {e}")
            raise ValueError(msg) from e
877

878
879
        # If opening a file fails, set chat template to be args to
        # ensure we decode so our escape are interpreted correctly
880
881
882
883
884
885
886
887
888
889
890
891
        return _load_chat_template(chat_template, is_literal=True)


_cached_load_chat_template = lru_cache(_load_chat_template)


def load_chat_template(
    chat_template: Optional[Union[Path, str]],
    *,
    is_literal: bool = False,
) -> Optional[str]:
    return _cached_load_chat_template(chat_template, is_literal=is_literal)
892
893


894
# TODO: Let user specify how to insert multimodal tokens into prompt
895
# (similar to chat template)
896
def _get_full_multimodal_text_prompt(placeholder_counts: dict[str, int],
897
                                     text_prompt: str) -> str:
898
    """Combine multimodal prompts for a multimodal language model."""
899

900
    # Look through the text prompt to check for missing placeholders
901
    missing_placeholders: list[str] = []
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
    for placeholder in placeholder_counts:

        # For any existing placeholder in the text prompt, we leave it as is
        placeholder_counts[placeholder] -= text_prompt.count(placeholder)

        if placeholder_counts[placeholder] < 0:
            raise ValueError(
                f"Found more '{placeholder}' placeholders in input prompt than "
                "actual multimodal data items.")

        missing_placeholders.extend([placeholder] *
                                    placeholder_counts[placeholder])

    # NOTE: For now we always add missing placeholders at the front of
    # the prompt. This may change to be customizable in the future.
    return "\n".join(missing_placeholders + [text_prompt])
918
919


920
921
# No need to validate using Pydantic again
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
922
_ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
923
_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
924
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
925
926
927
928
# Need to validate url objects
_ImageParser = TypeAdapter(ChatCompletionContentPartImageParam).validate_python
_AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
_VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python
929

930
_ContentPart: TypeAlias = Union[str, dict[str, str], InputAudio]
931

932
# Define a mapping from part types to their corresponding parsing functions.
933
MM_PARSER_MAP: dict[
934
935
936
    str,
    Callable[[ChatCompletionContentPartParam], _ContentPart],
] = {
937
    "text":
938
    lambda part: _TextParser(part).get("text", None),
939
    "image_url":
940
    lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
941
    "image_embeds":
942
    lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
943
    "audio_url":
944
    lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
945
    "input_audio":
946
    lambda part: _InputAudioParser(part).get("input_audio", None),
947
    "refusal":
948
    lambda part: _RefusalParser(part).get("refusal", None),
949
    "video_url":
950
    lambda part: _VideoParser(part).get("video_url", {}).get("url", None),
951
952
953
954
}


def _parse_chat_message_content_mm_part(
955
        part: ChatCompletionContentPartParam) -> tuple[str, _ContentPart]:
956
    """
957
    Parses a given multi-modal content part based on its type.
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977

    Args:
        part: A dict containing the content part, with a potential 'type' field.

    Returns:
        A tuple (part_type, content) where:
        - part_type: Type of the part (e.g., 'text', 'image_url').
        - content: Parsed content (e.g., text, image URL).

    Raises:
        ValueError: If the 'type' field is missing and no direct URL is found.
    """
    assert isinstance(
        part, dict)  # This is needed to avoid mypy errors: part.get() from str
    part_type = part.get("type", None)

    if isinstance(part_type, str) and part_type in MM_PARSER_MAP:
        content = MM_PARSER_MAP[part_type](part)

        # Special case for 'image_url.detail'
978
979
        # We only support 'auto', which is the default
        if part_type == "image_url" and part.get("detail", "auto") != "auto":
980
981
982
983
984
985
            logger.warning("'image_url.detail' is currently not supported "
                           "and will be ignored.")

        return part_type, content

    # Handle missing 'type' but provided direct URL fields.
986
    # 'type' is required field by pydantic
987
988
989
990
991
992
993
994
995
    if part_type is None:
        if part.get("image_url") is not None:
            image_params = cast(CustomChatCompletionContentSimpleImageParam,
                                part)
            return "image_url", image_params.get("image_url", "")
        if part.get("audio_url") is not None:
            audio_params = cast(CustomChatCompletionContentSimpleAudioParam,
                                part)
            return "audio_url", audio_params.get("audio_url", "")
996
        if part.get("input_audio") is not None:
997
            input_audio_params = cast(dict[str, str], part)
998
            return "input_audio", input_audio_params
999
1000
1001
1002
        if part.get("video_url") is not None:
            video_params = cast(CustomChatCompletionContentSimpleVideoParam,
                                part)
            return "video_url", video_params.get("video_url", "")
1003
1004
1005
1006
1007
1008
1009
1010
1011
        # Raise an error if no 'type' or direct URL is found.
        raise ValueError("Missing 'type' field in multimodal part.")

    if not isinstance(part_type, str):
        raise ValueError("Invalid 'type' field in multimodal part.")
    return part_type, "unknown part_type content"


VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",
1012
                                       "image_embeds",
1013
                                       "audio_url", "input_audio", "video_url")
1014

1015

1016
1017
1018
def _parse_chat_message_content_parts(
    role: str,
    parts: Iterable[ChatCompletionContentPartParam],
1019
    mm_tracker: BaseMultiModalItemTracker,
1020
1021
    *,
    wrap_dicts: bool,
1022
) -> list[ConversationMessage]:
1023
    content = list[_ContentPart]()
1024

1025
    mm_parser = mm_tracker.create_parser()
1026
1027

    for part in parts:
1028
        parse_res = _parse_chat_message_content_part(
1029
1030
1031
1032
            part,
            mm_parser,
            wrap_dicts=wrap_dicts,
        )
1033
1034
        if parse_res:
            content.append(parse_res)
1035

1036
    if wrap_dicts:
1037
        # Parsing wraps images and texts as interleaved dictionaries
1038
        return [ConversationMessage(role=role,
1039
                                    content=content)]  # type: ignore
1040
    texts = cast(list[str], content)
1041
1042
1043
1044
1045
1046
1047
1048
1049
    text_prompt = "\n".join(texts)
    mm_placeholder_counts = mm_parser.mm_placeholder_counts()
    if mm_placeholder_counts:
        text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
                                                       text_prompt)
    return [ConversationMessage(role=role, content=text_prompt)]


def _parse_chat_message_content_part(
1050
1051
1052
1053
    part: ChatCompletionContentPartParam,
    mm_parser: BaseMultiModalContentParser,
    *,
    wrap_dicts: bool,
1054
) -> Optional[_ContentPart]:
1055
1056
1057
1058
1059
1060
1061
1062
    """Parses a single part of a conversation. If wrap_dicts is True,
    structured dictionary pieces for texts and images will be
    wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
    {"type": "image"}, respectively. Otherwise multimodal data will be
    handled by mm_parser, and texts will be returned as strings to be joined
    with multimodal placeholders.
    """
    if isinstance(part, str):  # Handle plain text parts
1063
        return part
1064
1065
1066
1067

    # Handle structured dictionary parts
    part_type, content = _parse_chat_message_content_mm_part(part)

1068
    # if part_type is text/refusal/image_url/audio_url/video_url/input_audio but
1069
1070
    # content is None, log a warning and skip
    if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and content is None:
1071
        logger.warning(
1072
1073
            "Skipping multimodal part '%s' (type: '%s') "
            "with empty / unparsable content.", part, part_type)
1074
1075
1076
        return None

    if part_type in ("text", "refusal"):
1077
1078
1079
1080
1081
        str_content = cast(str, content)
        if wrap_dicts:
            return {'type': 'text', 'text': str_content}
        else:
            return str_content
1082
1083

    if part_type == "image_url":
1084
1085
        str_content = cast(str, content)
        mm_parser.parse_image(str_content)
1086
        return {'type': 'image'} if wrap_dicts else None
1087
1088
1089
1090
    if part_type == "image_embeds":
        content = cast(Union[str, dict[str, str]], content)
        mm_parser.parse_image_embeds(content)
        return {'type': 'image'} if wrap_dicts else None
1091
    if part_type == "audio_url":
1092
1093
1094
1095
1096
        str_content = cast(str, content)
        mm_parser.parse_audio(str_content)
        return {'type': 'audio'} if wrap_dicts else None

    if part_type == "input_audio":
1097
        dict_content = cast(InputAudio, content)
1098
        mm_parser.parse_input_audio(dict_content)
1099
1100
        return {'type': 'audio'} if wrap_dicts else None

1101
    if part_type == "video_url":
1102
1103
        str_content = cast(str, content)
        mm_parser.parse_video(str_content)
1104
1105
        return {'type': 'video'} if wrap_dicts else None

1106
    raise NotImplementedError(f"Unknown part type: {part_type}")
1107
1108


1109
1110
1111
1112
1113
# No need to validate using Pydantic again
_AssistantParser = partial(cast, ChatCompletionAssistantMessageParam)
_ToolParser = partial(cast, ChatCompletionToolMessageParam)


1114
def _parse_chat_message_content(
1115
1116
    message: ChatCompletionMessageParam,
    mm_tracker: BaseMultiModalItemTracker,
1117
    content_format: _ChatTemplateContentFormat,
1118
) -> list[ConversationMessage]:
1119
1120
1121
1122
    role = message["role"]
    content = message.get("content")

    if content is None:
1123
1124
1125
1126
1127
1128
        content = []
    elif isinstance(content, str):
        content = [
            ChatCompletionContentPartTextParam(type="text", text=content)
        ]
    result = _parse_chat_message_content_parts(
1129
1130
        role,
        content,  # type: ignore
1131
        mm_tracker,
1132
        wrap_dicts=(content_format == "openai"),
1133
    )
1134

1135
1136
1137
1138
    for result_msg in result:
        if role == 'assistant':
            parsed_msg = _AssistantParser(message)

1139
1140
1141
1142
1143
            # The 'tool_calls' is not None check ensures compatibility.
            # It's needed only if downstream code doesn't strictly
            # follow the OpenAI spec.
            if ("tool_calls" in parsed_msg
                and parsed_msg["tool_calls"] is not None):
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
                result_msg["tool_calls"] = list(parsed_msg["tool_calls"])
        elif role == "tool":
            parsed_msg = _ToolParser(message)
            if "tool_call_id" in parsed_msg:
                result_msg["tool_call_id"] = parsed_msg["tool_call_id"]

        if "name" in message and isinstance(message["name"], str):
            result_msg["name"] = message["name"]

    return result

1155

1156
def _postprocess_messages(messages: list[ConversationMessage]) -> None:
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
    # per the Transformers docs & maintainers, tool call arguments in
    # assistant-role messages with tool_calls need to be dicts not JSON str -
    # this is how tool-use chat templates will expect them moving forwards
    # so, for messages that have tool_calls, parse the string (which we get
    # from openAI format) to dict
    for message in messages:
        if (message["role"] == "assistant" and "tool_calls" in message
                and isinstance(message["tool_calls"], list)):

            for item in message["tool_calls"]:
                item["function"]["arguments"] = json.loads(
                    item["function"]["arguments"])


1171
def parse_chat_messages(
1172
    messages: list[ChatCompletionMessageParam],
1173
    model_config: ModelConfig,
1174
    tokenizer: AnyTokenizer,
1175
    content_format: _ChatTemplateContentFormat,
1176
1177
) -> tuple[list[ConversationMessage], Optional[MultiModalDataDict]]:
    conversation: list[ConversationMessage] = []
1178
    mm_tracker = MultiModalItemTracker(model_config, tokenizer)
1179
1180

    for msg in messages:
1181
1182
1183
        sub_messages = _parse_chat_message_content(
            msg,
            mm_tracker,
1184
            content_format,
1185
        )
1186

1187
        conversation.extend(sub_messages)
1188

1189
1190
    _postprocess_messages(conversation)

1191
    return conversation, mm_tracker.all_mm_data()
1192
1193


1194
def parse_chat_messages_futures(
1195
    messages: list[ChatCompletionMessageParam],
1196
1197
    model_config: ModelConfig,
    tokenizer: AnyTokenizer,
1198
    content_format: _ChatTemplateContentFormat,
1199
1200
) -> tuple[list[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:
    conversation: list[ConversationMessage] = []
1201
1202
1203
    mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)

    for msg in messages:
1204
1205
1206
        sub_messages = _parse_chat_message_content(
            msg,
            mm_tracker,
1207
            content_format,
1208
        )
1209
1210
1211

        conversation.extend(sub_messages)

1212
1213
    _postprocess_messages(conversation)

1214
1215
1216
    return conversation, mm_tracker.all_mm_data()


1217
1218
1219
1220
@deprecate_kwargs(
    "trust_remote_code",
    additional_message="Please use `model_config.trust_remote_code` instead.",
)
1221
1222
def apply_hf_chat_template(
    tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
1223
    conversation: list[ConversationMessage],
1224
    chat_template: Optional[str],
1225
    tools: Optional[list[dict[str, Any]]],
1226
    *,
1227
    model_config: ModelConfig,
1228
    tokenize: bool = False,  # Different from HF's default
1229
1230
    # Deprecated, explicitly capture here so it doesn't slit into kwargs.
    trust_remote_code: Optional[bool] = None,
1231
    **kwargs: Any,
1232
) -> str:
1233
    hf_chat_template = resolve_hf_chat_template(
1234
1235
1236
        tokenizer,
        chat_template=chat_template,
        tools=tools,
1237
        model_config=model_config,
1238
    )
1239

1240
    if hf_chat_template is None:
1241
1242
1243
1244
1245
        raise ValueError(
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one.")

1246
1247
1248
1249
1250
1251
1252
1253
1254
    try:

        return tokenizer.apply_chat_template(
            conversation=conversation,  # type: ignore[arg-type]
            tools=tools,  # type: ignore[arg-type]
            chat_template=hf_chat_template,
            tokenize=tokenize,
            **kwargs,
        )
1255

1256
1257
1258
1259
1260
1261
1262
1263
    # External library exceptions can sometimes occur despite the framework's
    # internal exception management capabilities.
    except Exception as e:

        # Log and report any library-related exceptions for further
        # investigation.
        logger.exception(
            "An error occurred in `transformers` while applying chat template")
1264
        raise ValueError(str(e)) from e
1265

1266
1267
def apply_mistral_chat_template(
    tokenizer: MistralTokenizer,
1268
    messages: list[ChatCompletionMessageParam],
1269
1270
    chat_template: Optional[str],
    tools: Optional[list[dict[str, Any]]],
1271
    **kwargs: Any,
1272
) -> list[int]:
1273
1274
    from mistral_common.exceptions import MistralCommonException

1275
1276
1277
1278
1279
1280
    # The return value of resolve_mistral_chat_template is always None,
    # and we won't use it.
    resolve_mistral_chat_template(
        chat_template=chat_template,
        **kwargs,
    )
1281

1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
    try:
        return tokenizer.apply_chat_template(
            messages=messages,
            tools=tools,
            **kwargs,
        )
    # mistral-common uses assert statements to stop processing of input
    # if input does not comply with the expected format.
    # We convert those assertion errors to ValueErrors so they can be
    # are properly caught in the preprocessing_input step
1292
    except (AssertionError, MistralCommonException) as e:
1293
        raise ValueError(str(e)) from e
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303

    # External library exceptions can sometimes occur despite the framework's
    # internal exception management capabilities.
    except Exception as e:

        # Log and report any library-related exceptions for further
        # investigation.
        logger.exception(
            "An error occurred in `mistral_common` while applying chat "
            "template")
1304
        raise ValueError(str(e)) from e
1305
1306

def random_tool_call_id() -> str:
1307
    return f"chatcmpl-tool-{random_uuid()}"