"tests/entrypoints/openai/test_chat_template.py" did not exist on "66d617e3437e62a6650ffcb85b3190669d37a468"
chat_utils.py 44.3 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import asyncio
4
import json
5
from abc import ABC, abstractmethod
6
from collections import defaultdict, deque
7
from collections.abc import Awaitable, Iterable
8
from functools import cache, lru_cache, partial
9
from pathlib import Path
10
11
from typing import (Any, Callable, Generic, Literal, Optional, TypeVar, Union,
                    cast)
12

13
14
import jinja2.nodes
import transformers.utils.chat_template_utils as hf_chat_utils
15
16
# yapf conflicts with isort for this block
# yapf: disable
17
from openai.types.chat import (ChatCompletionAssistantMessageParam,
18
19
                               ChatCompletionContentPartImageParam,
                               ChatCompletionContentPartInputAudioParam)
20
21
from openai.types.chat import (
    ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam)
22
23
from openai.types.chat import (ChatCompletionContentPartRefusalParam,
                               ChatCompletionContentPartTextParam)
24
25
from openai.types.chat import (
    ChatCompletionMessageParam as OpenAIChatCompletionMessageParam)
26
27
from openai.types.chat import (ChatCompletionMessageToolCallParam,
                               ChatCompletionToolMessageParam)
28
29
from openai.types.chat.chat_completion_content_part_input_audio_param import (
    InputAudio)
30
from pydantic import TypeAdapter
31
# yapf: enable
32
33
from transformers import (PreTrainedTokenizer, PreTrainedTokenizerFast,
                          ProcessorMixin)
34
# pydantic needs the TypedDict from typing_extensions
35
from typing_extensions import Required, TypeAlias, TypedDict
36

37
from vllm.config import ModelConfig
38
from vllm.logger import init_logger
39
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
40
from vllm.multimodal.utils import MediaConnector
41
from vllm.transformers_utils.processor import cached_get_processor
42
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
43
44
45
46

logger = init_logger(__name__)


47
48
49
50
51
52
53
54
55
56
57
58
59
60
class AudioURL(TypedDict, total=False):
    url: Required[str]
    """
    Either a URL of the audio or a data URL with base64 encoded audio data.
    """


class ChatCompletionContentPartAudioParam(TypedDict, total=False):
    audio_url: Required[AudioURL]

    type: Required[Literal["audio_url"]]
    """The type of the content part."""


61
62
63
64
65
66
67
68
69
70
71
class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
    image_embeds: Required[Union[str, dict[str, str]]]
    """
    The image embeddings. It can be either:
    - A single base64 string.
    - A dictionary where each value is a base64 string.
    """
    type: Required[Literal["image_embeds"]]
    """The type of the content part."""


72
73
74
75
76
77
78
79
80
81
82
83
84
85
class VideoURL(TypedDict, total=False):
    url: Required[str]
    """
    Either a URL of the video or a data URL with base64 encoded video data.
    """


class ChatCompletionContentPartVideoParam(TypedDict, total=False):
    video_url: Required[VideoURL]

    type: Required[Literal["video_url"]]
    """The type of the content part."""


86
87
88
class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain image_url.
    This is supported by OpenAI API, although it is not documented.
89

90
91
92
93
94
95
96
97
98
99
    Example:
    {
        "image_url": "https://example.com/image.jpg"
    }
    """
    image_url: Required[str]


class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain audio_url.
100

101
102
103
104
105
106
107
108
    Example:
    {
        "audio_url": "https://example.com/audio.mp3"
    }
    """
    audio_url: Required[str]


109
110
111
112
113
114
115
116
117
118
119
class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain audio_url.

    Example:
    {
        "video_url": "https://example.com/video.mp4"
    }
    """
    video_url: Required[str]


120
121
ChatCompletionContentPartParam: TypeAlias = Union[
    OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
122
    ChatCompletionContentPartInputAudioParam,
123
    ChatCompletionContentPartVideoParam, ChatCompletionContentPartRefusalParam,
124
    CustomChatCompletionContentSimpleImageParam,
125
    ChatCompletionContentPartImageEmbedsParam,
126
127
    CustomChatCompletionContentSimpleAudioParam,
    CustomChatCompletionContentSimpleVideoParam, str]
128
129
130
131
132
133
134


class CustomChatCompletionMessageParam(TypedDict, total=False):
    """Enables custom roles in the Chat Completion API."""
    role: Required[str]
    """The role of the message's author."""

135
    content: Union[str, list[ChatCompletionContentPartParam]]
136
137
138
139
140
141
142
143
144
    """The contents of the message."""

    name: str
    """An optional name for the participant.

    Provides the model information to differentiate between participants of the
    same role.
    """

145
146
147
148
149
150
    tool_call_id: Optional[str]
    """Tool call that this message is responding to."""

    tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
    """The tool calls generated by the model, such as function calls."""

151
152
153
154
155

ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam,
                                   CustomChatCompletionMessageParam]


156
# TODO: Make fields ReadOnly once mypy supports it
157
158
159
160
class ConversationMessage(TypedDict, total=False):
    role: Required[str]
    """The role of the message's author."""

161
    content: Union[Optional[str], list[dict[str, str]]]
162
163
164
165
166
167
168
169
170
171
    """The contents of the message"""

    tool_call_id: Optional[str]
    """Tool call that this message is responding to."""

    name: Optional[str]
    """The name of the function to call"""

    tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
    """The tool calls generated by the model, such as function calls."""
172
173


174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
# Passed in by user
ChatTemplateContentFormatOption = Literal["auto", "string", "openai"]

# Used internally
_ChatTemplateContentFormat = Literal["string", "openai"]


def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool:
    if isinstance(node, jinja2.nodes.Name):
        return node.ctx == "load" and node.name == varname

    return False


def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
    if isinstance(node, jinja2.nodes.Getitem):
        return (_is_var_access(node.node, varname)
                and isinstance(node.arg, jinja2.nodes.Const)
                and node.arg.value == key)

    if isinstance(node, jinja2.nodes.Getattr):
        return _is_var_access(node.node, varname) and node.attr == key

    return False


def _is_var_or_elems_access(
    node: jinja2.nodes.Node,
    varname: str,
    key: Optional[str] = None,
) -> bool:
    if isinstance(node, jinja2.nodes.Filter):
        return (node.node is not None
                and _is_var_or_elems_access(node.node, varname, key))
    if isinstance(node, jinja2.nodes.Test):
        return _is_var_or_elems_access(node.node, varname, key)

    if (isinstance(node, jinja2.nodes.Getitem)
            and isinstance(node.arg, jinja2.nodes.Slice)):
        return _is_var_or_elems_access(node.node, varname, key)

    # yapf: disable
    return (
        _is_attr_access(node, varname, key) if key
        else _is_var_access(node, varname)
    ) # yapf: enable


def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str):
    # Global variable that is implicitly defined at the root
    yield root, varname

    # Iterative BFS
    related_varnames = deque([varname])
    while related_varnames:
        related_varname = related_varnames.popleft()

        for assign_ast in root.find_all(jinja2.nodes.Assign):
            lhs = assign_ast.target
            rhs = assign_ast.node

            if _is_var_or_elems_access(rhs, related_varname):
                assert isinstance(lhs, jinja2.nodes.Name)
                yield assign_ast, lhs.name

                # Avoid infinite looping for self-assignment
                if lhs.name != related_varname:
                    related_varnames.append(lhs.name)


# NOTE: The proper way to handle this is to build a CFG so that we can handle
# the scope in which each variable is defined, but that is too complicated
def _iter_nodes_assign_messages_item(root: jinja2.nodes.Node):
    messages_varnames = [
        varname
        for _, varname in _iter_nodes_assign_var_or_elems(root, "messages")
    ]

    # Search for {%- for message in messages -%} loops
    for loop_ast in root.find_all(jinja2.nodes.For):
        loop_iter = loop_ast.iter
        loop_target = loop_ast.target

        for varname in messages_varnames:
            if _is_var_or_elems_access(loop_iter, varname):
                assert isinstance(loop_target, jinja2.nodes.Name)
                yield loop_ast, loop_target.name
                break


def _iter_nodes_assign_content_item(root: jinja2.nodes.Node):
    message_varnames = [
        varname for _, varname in _iter_nodes_assign_messages_item(root)
    ]

    # Search for {%- for content in message['content'] -%} loops
    for loop_ast in root.find_all(jinja2.nodes.For):
        loop_iter = loop_ast.iter
        loop_target = loop_ast.target

        for varname in message_varnames:
            if _is_var_or_elems_access(loop_iter, varname, "content"):
                assert isinstance(loop_target, jinja2.nodes.Name)
                yield loop_ast, loop_target.name
                break


def _try_extract_ast(chat_template: str) -> Optional[jinja2.nodes.Template]:
    try:
        jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
        return jinja_compiled.environment.parse(chat_template)
    except Exception:
        logger.exception("Error when compiling Jinja template")
        return None


def _detect_content_format(
    chat_template: str,
    *,
    default: _ChatTemplateContentFormat,
) -> _ChatTemplateContentFormat:
    jinja_ast = _try_extract_ast(chat_template)
    if jinja_ast is None:
        return default

    try:
        next(_iter_nodes_assign_content_item(jinja_ast))
    except StopIteration:
        return "string"
    except Exception:
        logger.exception("Error when parsing AST of Jinja template")
        return default
    else:
        return "openai"


310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
def resolve_mistral_chat_template(
    chat_template: Optional[str],
    **kwargs: Any,
) -> Optional[str]:
    if chat_template is not None:
        logger.warning_once(
            "'chat_template' cannot be overridden for mistral tokenizer.")
    if "add_generation_prompt" in kwargs:
        logger.warning_once(
            "'add_generation_prompt' is not supported for mistral tokenizer, "
            "so it will be ignored.")
    if "continue_final_message" in kwargs:
        logger.warning_once(
            "'continue_final_message' is not supported for mistral tokenizer, "
            "so it will be ignored.")
    return None

def resolve_hf_chat_template(
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
    tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
    chat_template: Optional[str],
    tools: Optional[list[dict[str, Any]]],
    *,
    trust_remote_code: bool,
) -> Optional[str]:
    # 1st priority: The given chat template
    if chat_template is not None:
        return chat_template

    # 2nd priority: AutoProcessor chat template, unless tool calling is enabled
    if tools is None:
        try:
            processor = cached_get_processor(
                tokenizer.name_or_path,
                processor_cls=(PreTrainedTokenizer, PreTrainedTokenizerFast,
                               ProcessorMixin),
                trust_remote_code=trust_remote_code,
            )
            if isinstance(processor, ProcessorMixin) and \
                processor.chat_template is not None:
                return processor.chat_template
        except Exception:
            logger.debug("Failed to load AutoProcessor chat template for %s",
                        tokenizer.name_or_path, exc_info=True)

    # 3rd priority: AutoTokenizer chat template
    try:
        return tokenizer.get_chat_template(chat_template, tools=tools)
    except Exception:
        logger.debug("Failed to load AutoTokenizer chat template for %s",
                     tokenizer.name_or_path, exc_info=True)

    return None


364
365
def _resolve_chat_template_content_format(
    chat_template: Optional[str],
366
    tools: Optional[list[dict[str, Any]]],
367
368
    given_format: ChatTemplateContentFormatOption,
    tokenizer: AnyTokenizer,
369
370
    *,
    trust_remote_code: bool,
371
372
) -> _ChatTemplateContentFormat:
    if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
373
        hf_chat_template = resolve_hf_chat_template(
374
375
376
377
378
            tokenizer,
            chat_template=chat_template,
            trust_remote_code=trust_remote_code,
            tools=tools,
        )
379
    else:
380
381
382
383
        hf_chat_template = None

    jinja_text = (hf_chat_template if isinstance(hf_chat_template, str)
                  else load_chat_template(chat_template, is_literal=True))
384
385
386
387
388
389
390
391

    detected_format = ("string" if jinja_text is None else
                       _detect_content_format(jinja_text, default="string"))

    return detected_format if given_format == "auto" else given_format


@lru_cache
392
def _log_chat_template_content_format(
393
394
    chat_template: Optional[str],
    given_format: ChatTemplateContentFormatOption,
395
396
    detected_format: ChatTemplateContentFormatOption,
):
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
    logger.info(
        "Detected the chat template content format to be '%s'. "
        "You can set `--chat-template-content-format` to override this.",
        detected_format,
    )

    if given_format != "auto" and given_format != detected_format:
        logger.warning(
            "You specified `--chat-template-content-format %s` "
            "which is different from the detected format '%s'. "
            "If our automatic detection is incorrect, please consider "
            "opening a GitHub issue so that we can improve it: "
            "https://github.com/vllm-project/vllm/issues/new/choose",
            given_format,
            detected_format,
        )

414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436

def resolve_chat_template_content_format(
    chat_template: Optional[str],
    tools: Optional[list[dict[str, Any]]],
    given_format: ChatTemplateContentFormatOption,
    tokenizer: AnyTokenizer,
    *,
    trust_remote_code: bool = False,
) -> _ChatTemplateContentFormat:
    detected_format = _resolve_chat_template_content_format(
        chat_template,
        tools,
        given_format,
        tokenizer,
        trust_remote_code=trust_remote_code,
    )

    _log_chat_template_content_format(
        chat_template,
        given_format=given_format,
        detected_format=detected_format,
    )

437
438
439
    return detected_format


440
ModalityStr = Literal["image", "audio", "video", "image_embeds"]
441
442
443
444
_T = TypeVar("_T")


class BaseMultiModalItemTracker(ABC, Generic[_T]):
445
446
447
448
449
450
451
    """
    Tracks multi-modal items in a given request and ensures that the number
    of multi-modal items in a given request does not exceed the configured
    maximum per prompt.
    """

    def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer):
452
453
        super().__init__()

454
455
        self._model_config = model_config
        self._tokenizer = tokenizer
456

457
        self._items_by_modality = defaultdict[str, list[_T]](list)
458

459
460
461
462
    @property
    def model_config(self) -> ModelConfig:
        return self._model_config

463
464
465
466
    @property
    def allowed_local_media_path(self):
        return self._model_config.allowed_local_media_path

467
468
469
470
    @property
    def mm_registry(self):
        return MULTIMODAL_REGISTRY

471
    @staticmethod
472
    @cache
473
    def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
474
475
        return tokenizer.decode(token_index)

476
477
    def _placeholder_str(self, modality: ModalityStr,
                         current_count: int) -> Optional[str]:
478
479
        # TODO: Let user specify how to insert image tokens into prompt
        # (similar to chat template)
480
481
482
        hf_config = self._model_config.hf_config
        model_type = hf_config.model_type

483
484
485
        if modality in ("image", "image_embeds"):
            if model_type == "chatglm":
                return "<|begin_of_image|><|endoftext|><|end_of_image|>"
486
            if model_type in ("phi3_v", "phi4mm"):
487
                return f"<|image_{current_count}|>"
488
            if model_type in ("minicpmo", "minicpmv"):
489
                return "(<image>./</image>)"
490
491
            if model_type in ("blip-2", "florence2", "fuyu", "paligemma",
                              "pixtral", "mistral3"):
492
493
                # These models do not use image tokens in the prompt
                return None
494
495
            if model_type == "qwen":
                return f"Picture {current_count}: <img></img>"
496
            if model_type.startswith("llava"):
497
498
                return self._cached_token_str(self._tokenizer,
                                              hf_config.image_token_index)
499

Jennifer Zhao's avatar
Jennifer Zhao committed
500
            if model_type in ("aya_vision", "chameleon", "deepseek_vl_v2",
501
502
                              "internvl_chat", "ovis2", "skywork_chat",
                              "NVLM_D", "h2ovl_chat", "idefics3", "smolvlm"):
503
                return "<image>"
504
            if model_type in ("mllama", "llama4"):
505
                return "<|image|>"
Roger Wang's avatar
Roger Wang committed
506
            if model_type in ("qwen2_vl", "qwen2_5_vl"):
507
                return "<|vision_start|><|image_pad|><|vision_end|>"
508
509
            if model_type == "qwen2_5_omni":
                return "<|vision_start|><|IMAGE|><|vision_end|>"
510
511
            if model_type == "molmo":
                return ""
512
513
            if model_type == "aria":
                return "<|fim_prefix|><|img|><|fim_suffix|>"
514
515
            if model_type == "gemma3":
                return "<start_of_image>"
516
517
            if model_type == "kimi_vl":
                return "<|media_start|>image<|media_content|><|media_pad|><|media_end|>" # noqa: E501
518

519
            raise TypeError(f"Unknown {modality} model type: {model_type}")
520
        elif modality == "audio":
521
            if model_type in ("ultravox", "granite_speech"):
522
                return "<|audio|>"
523
            if model_type == "phi4mm":
524
                return f"<|audio_{current_count}|>"
525
            if model_type in ("qwen2_audio", "qwen2_5_omni"):
526
527
                return (f"Audio {current_count}: "
                        f"<|audio_bos|><|AUDIO|><|audio_eos|>")
528
529
            if model_type == "minicpmo":
                return "(<audio>./</audio>)"
530
            raise TypeError(f"Unknown model type: {model_type}")
531
        elif modality == "video":
Roger Wang's avatar
Roger Wang committed
532
            if model_type in ("qwen2_vl", "qwen2_5_vl"):
533
                return "<|vision_start|><|video_pad|><|vision_end|>"
534
535
            if model_type == "qwen2_5_omni":
                return "<|vision_start|><|VIDEO|><|vision_end|>"
536
537
            if model_type in ("minicpmo", "minicpmv"):
                return "(<video>./</video>)"
538
539
540
            if model_type.startswith("llava"):
                return self._cached_token_str(self._tokenizer,
                                              hf_config.video_token_index)
541
            raise TypeError(f"Unknown {modality} model type: {model_type}")
542
543
544
        else:
            raise TypeError(f"Unknown modality: {modality}")

545
546
547
548
549
    def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
        """
        Add a multi-modal item to the current prompt and returns the
        placeholder string to use, if any.
        """
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
        mm_registry = self.mm_registry
        model_config = self.model_config

        input_modality = modality.replace("_embeds", "")

        if mm_registry.has_processor(model_config):
            mm_processor = mm_registry.create_processor(model_config)
            allowed_counts = mm_processor.info.get_allowed_mm_limits()
            allowed_count = allowed_counts.get(input_modality, 0)
        else:
            mm_config = model_config.multimodal_config
            if mm_config is None:
                msg = "This model does not support multi-modal inputs"
                raise ValueError(msg)

            allowed_count = mm_config.get_limit_per_prompt(input_modality)

567
        current_count = len(self._items_by_modality[modality]) + 1
568
569
570
        if current_count > allowed_count:
            raise ValueError(
                f"At most {allowed_count} {modality}(s) may be provided in "
571
572
                "one request. You can set `--limit-mm-per-prompt` to "
                "increase this limit if the model supports it.")
573

574
        self._items_by_modality[modality].append(item)
575
576
577
578
579
580
581
582

        return self._placeholder_str(modality, current_count)

    @abstractmethod
    def create_parser(self) -> "BaseMultiModalContentParser":
        raise NotImplementedError


583
class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
584
585

    def all_mm_data(self) -> Optional[MultiModalDataDict]:
586
587
588
589
590
591
592
593
594
595
596
597
598
599
        if not self._items_by_modality:
            return None
        mm_inputs = {}
        items_by_modality = dict(self._items_by_modality)
        if "image" in items_by_modality and "image_embeds" in items_by_modality:
            raise ValueError(\
                "Mixing raw image and embedding inputs is not allowed")

        if "image_embeds" in items_by_modality:
            image_embeds_lst = items_by_modality["image_embeds"]
            if len(image_embeds_lst) > 1:
                raise ValueError(\
                    "Only one message can have {'type': 'image_embeds'}")
            mm_inputs["image"] = image_embeds_lst[0]
600
        if "image" in items_by_modality:
601
            mm_inputs["image"] = items_by_modality["image"] # A list of images
602
        if "audio" in items_by_modality:
603
            mm_inputs["audio"] = items_by_modality["audio"] # A list of audios
604
        if "video" in items_by_modality:
605
606
            mm_inputs["video"] = items_by_modality["video"] # A list of videos
        return mm_inputs
607
608
609
610
611

    def create_parser(self) -> "BaseMultiModalContentParser":
        return MultiModalContentParser(self)


612
class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
613
614

    async def all_mm_data(self) -> Optional[MultiModalDataDict]:
615
616
617
618
        if not self._items_by_modality:
            return None
        mm_inputs = {}
        items_by_modality = {
619
620
621
                modality: await asyncio.gather(*items)
                for modality, items in self._items_by_modality.items()
            }
622

623
624
625
626
627
628
629
630
631
632
        if "image" in items_by_modality and "image_embeds" in items_by_modality:
            raise ValueError(
                "Mixing raw image and embedding inputs is not allowed")

        if "image_embeds" in items_by_modality:
            image_embeds_lst = items_by_modality["image_embeds"]
            if len(image_embeds_lst) > 1:
                raise ValueError(
                    "Only one message can have {'type': 'image_embeds'}")
            mm_inputs["image"] = image_embeds_lst[0]
633
        if "image" in items_by_modality:
634
            mm_inputs["image"] = items_by_modality["image"] # A list of images
635
        if "audio" in items_by_modality:
636
            mm_inputs["audio"] = items_by_modality["audio"] # A list of audios
637
        if "video" in items_by_modality:
638
639
            mm_inputs["video"] = items_by_modality["video"] # A list of videos
        return mm_inputs
640
641
642
643
644
645
646
647
648
649
650

    def create_parser(self) -> "BaseMultiModalContentParser":
        return AsyncMultiModalContentParser(self)


class BaseMultiModalContentParser(ABC):

    def __init__(self) -> None:
        super().__init__()

        # multimodal placeholder_string : count
651
        self._placeholder_counts: dict[str, int] = defaultdict(lambda: 0)
652
653
654
655
656

    def _add_placeholder(self, placeholder: Optional[str]):
        if placeholder:
            self._placeholder_counts[placeholder] += 1

657
    def mm_placeholder_counts(self) -> dict[str, int]:
658
659
660
661
662
663
        return dict(self._placeholder_counts)

    @abstractmethod
    def parse_image(self, image_url: str) -> None:
        raise NotImplementedError

664
665
666
667
668
    @abstractmethod
    def parse_image_embeds(self,
                           image_embeds: Union[str, dict[str, str]]) -> None:
        raise NotImplementedError

669
670
671
672
    @abstractmethod
    def parse_audio(self, audio_url: str) -> None:
        raise NotImplementedError

673
    @abstractmethod
674
    def parse_input_audio(self, input_audio: InputAudio) -> None:
675
676
        raise NotImplementedError

677
678
679
680
    @abstractmethod
    def parse_video(self, video_url: str) -> None:
        raise NotImplementedError

681
682
683
684
685
686
687
688

class MultiModalContentParser(BaseMultiModalContentParser):

    def __init__(self, tracker: MultiModalItemTracker) -> None:
        super().__init__()

        self._tracker = tracker

689
690
691
692
        self._connector = MediaConnector(
            allowed_local_media_path=tracker.allowed_local_media_path,
        )

693
    def parse_image(self, image_url: str) -> None:
694
        image = self._connector.fetch_image(image_url)
695
696
697
698

        placeholder = self._tracker.add("image", image)
        self._add_placeholder(placeholder)

699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
    def parse_image_embeds(self,
                           image_embeds: Union[str, dict[str, str]]) -> None:
        if isinstance(image_embeds, dict):
            embeds = {
                k: self._connector.fetch_image_embedding(v)
                for k, v in image_embeds.items()
            }
            placeholder = self._tracker.add("image_embeds", embeds)

        if isinstance(image_embeds, str):
            embedding = self._connector.fetch_image_embedding(image_embeds)
            placeholder = self._tracker.add("image_embeds", embedding)

        self._add_placeholder(placeholder)

714
    def parse_audio(self, audio_url: str) -> None:
715
        audio = self._connector.fetch_audio(audio_url)
716
717
718
719

        placeholder = self._tracker.add("audio", audio)
        self._add_placeholder(placeholder)

720
721
722
723
    def parse_input_audio(self, input_audio: InputAudio) -> None:
        audio_data = input_audio.get("data", "")
        audio_format = input_audio.get("format", "")
        audio_url = f"data:audio/{audio_format};base64,{audio_data}"
724

725
        return self.parse_audio(audio_url)
726

727
    def parse_video(self, video_url: str) -> None:
728
        video = self._connector.fetch_video(video_url)
729
730
731
732

        placeholder = self._tracker.add("video", video)
        self._add_placeholder(placeholder)

733
734
735
736
737
738
739

class AsyncMultiModalContentParser(BaseMultiModalContentParser):

    def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:
        super().__init__()

        self._tracker = tracker
740
741
742
        self._connector = MediaConnector(
            allowed_local_media_path=tracker.allowed_local_media_path,
        )
743
744

    def parse_image(self, image_url: str) -> None:
745
        image_coro = self._connector.fetch_image_async(image_url)
746
747
748
749

        placeholder = self._tracker.add("image", image_coro)
        self._add_placeholder(placeholder)

750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
    def parse_image_embeds(self,
                           image_embeds: Union[str, dict[str, str]]) -> None:
        future: asyncio.Future[Union[str, dict[str, str]]] = asyncio.Future()

        if isinstance(image_embeds, dict):
            embeds = {
                k: self._connector.fetch_image_embedding(v)
                for k, v in image_embeds.items()
            }
            future.set_result(embeds)

        if isinstance(image_embeds, str):
            embedding = self._connector.\
                fetch_image_embedding(image_embeds)
            future.set_result(embedding)

        placeholder = self._tracker.add("image_embeds", future)
        self._add_placeholder(placeholder)

769
    def parse_audio(self, audio_url: str) -> None:
770
        audio_coro = self._connector.fetch_audio_async(audio_url)
771
772
773

        placeholder = self._tracker.add("audio", audio_coro)
        self._add_placeholder(placeholder)
774

775
776
777
778
    def parse_input_audio(self, input_audio: InputAudio) -> None:
        audio_data = input_audio.get("data", "")
        audio_format = input_audio.get("format", "")
        audio_url = f"data:audio/{audio_format};base64,{audio_data}"
779

780
        return self.parse_audio(audio_url)
781

782
    def parse_video(self, video_url: str) -> None:
783
        video = self._connector.fetch_video_async(video_url)
784
785
786
787

        placeholder = self._tracker.add("video", video)
        self._add_placeholder(placeholder)

788

789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
def validate_chat_template(chat_template: Optional[Union[Path, str]]):
    """Raises if the provided chat template appears invalid."""
    if chat_template is None:
        return

    elif isinstance(chat_template, Path) and not chat_template.exists():
        raise FileNotFoundError(
            "the supplied chat template path doesn't exist")

    elif isinstance(chat_template, str):
        JINJA_CHARS = "{}\n"
        if not any(c in chat_template
                   for c in JINJA_CHARS) and not Path(chat_template).exists():
            raise ValueError(
                f"The supplied chat template string ({chat_template}) "
                f"appears path-like, but doesn't exist!")

    else:
        raise TypeError(
            f"{type(chat_template)} is not a valid chat template type")


811
def _load_chat_template(
812
813
814
815
    chat_template: Optional[Union[Path, str]],
    *,
    is_literal: bool = False,
) -> Optional[str]:
816
817
    if chat_template is None:
        return None
818
819
820
821
822
823

    if is_literal:
        if isinstance(chat_template, Path):
            raise TypeError("chat_template is expected to be read directly "
                            "from its value")

824
        return chat_template
825

826
    try:
827
        with open(chat_template) as f:
828
            return f.read()
829
    except OSError as e:
830
831
832
        if isinstance(chat_template, Path):
            raise

833
834
835
836
837
838
        JINJA_CHARS = "{}\n"
        if not any(c in chat_template for c in JINJA_CHARS):
            msg = (f"The supplied chat template ({chat_template}) "
                   f"looks like a file path, but it failed to be "
                   f"opened. Reason: {e}")
            raise ValueError(msg) from e
839

840
841
        # If opening a file fails, set chat template to be args to
        # ensure we decode so our escape are interpreted correctly
842
843
844
845
846
847
848
849
850
851
852
853
        return _load_chat_template(chat_template, is_literal=True)


_cached_load_chat_template = lru_cache(_load_chat_template)


def load_chat_template(
    chat_template: Optional[Union[Path, str]],
    *,
    is_literal: bool = False,
) -> Optional[str]:
    return _cached_load_chat_template(chat_template, is_literal=is_literal)
854
855


856
# TODO: Let user specify how to insert multimodal tokens into prompt
857
# (similar to chat template)
858
def _get_full_multimodal_text_prompt(placeholder_counts: dict[str, int],
859
                                     text_prompt: str) -> str:
860
    """Combine multimodal prompts for a multimodal language model."""
861

862
    # Look through the text prompt to check for missing placeholders
863
    missing_placeholders: list[str] = []
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
    for placeholder in placeholder_counts:

        # For any existing placeholder in the text prompt, we leave it as is
        placeholder_counts[placeholder] -= text_prompt.count(placeholder)

        if placeholder_counts[placeholder] < 0:
            raise ValueError(
                f"Found more '{placeholder}' placeholders in input prompt than "
                "actual multimodal data items.")

        missing_placeholders.extend([placeholder] *
                                    placeholder_counts[placeholder])

    # NOTE: For now we always add missing placeholders at the front of
    # the prompt. This may change to be customizable in the future.
    return "\n".join(missing_placeholders + [text_prompt])
880
881


882
883
# No need to validate using Pydantic again
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
884
_ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
885
_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
886
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
887
888
889
890
# Need to validate url objects
_ImageParser = TypeAdapter(ChatCompletionContentPartImageParam).validate_python
_AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
_VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python
891

892
_ContentPart: TypeAlias = Union[str, dict[str, str], InputAudio]
893

894
# Define a mapping from part types to their corresponding parsing functions.
895
MM_PARSER_MAP: dict[
896
897
898
    str,
    Callable[[ChatCompletionContentPartParam], _ContentPart],
] = {
899
    "text":
900
    lambda part: _TextParser(part).get("text", None),
901
    "image_url":
902
    lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
903
    "image_embeds":
904
    lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
905
    "audio_url":
906
    lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
907
    "input_audio":
908
    lambda part: _InputAudioParser(part).get("input_audio", None),
909
    "refusal":
910
    lambda part: _RefusalParser(part).get("refusal", None),
911
    "video_url":
912
    lambda part: _VideoParser(part).get("video_url", {}).get("url", None),
913
914
915
916
}


def _parse_chat_message_content_mm_part(
917
        part: ChatCompletionContentPartParam) -> tuple[str, _ContentPart]:
918
    """
919
    Parses a given multi-modal content part based on its type.
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939

    Args:
        part: A dict containing the content part, with a potential 'type' field.

    Returns:
        A tuple (part_type, content) where:
        - part_type: Type of the part (e.g., 'text', 'image_url').
        - content: Parsed content (e.g., text, image URL).

    Raises:
        ValueError: If the 'type' field is missing and no direct URL is found.
    """
    assert isinstance(
        part, dict)  # This is needed to avoid mypy errors: part.get() from str
    part_type = part.get("type", None)

    if isinstance(part_type, str) and part_type in MM_PARSER_MAP:
        content = MM_PARSER_MAP[part_type](part)

        # Special case for 'image_url.detail'
940
941
        # We only support 'auto', which is the default
        if part_type == "image_url" and part.get("detail", "auto") != "auto":
942
943
944
945
946
947
            logger.warning("'image_url.detail' is currently not supported "
                           "and will be ignored.")

        return part_type, content

    # Handle missing 'type' but provided direct URL fields.
948
    # 'type' is required field by pydantic
949
950
951
952
953
954
955
956
957
    if part_type is None:
        if part.get("image_url") is not None:
            image_params = cast(CustomChatCompletionContentSimpleImageParam,
                                part)
            return "image_url", image_params.get("image_url", "")
        if part.get("audio_url") is not None:
            audio_params = cast(CustomChatCompletionContentSimpleAudioParam,
                                part)
            return "audio_url", audio_params.get("audio_url", "")
958
        if part.get("input_audio") is not None:
959
            input_audio_params = cast(dict[str, str], part)
960
            return "input_audio", input_audio_params
961
962
963
964
        if part.get("video_url") is not None:
            video_params = cast(CustomChatCompletionContentSimpleVideoParam,
                                part)
            return "video_url", video_params.get("video_url", "")
965
966
967
968
969
970
971
972
973
        # Raise an error if no 'type' or direct URL is found.
        raise ValueError("Missing 'type' field in multimodal part.")

    if not isinstance(part_type, str):
        raise ValueError("Invalid 'type' field in multimodal part.")
    return part_type, "unknown part_type content"


VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",
974
                                       "image_embeds",
975
                                       "audio_url", "input_audio", "video_url")
976

977

978
979
980
def _parse_chat_message_content_parts(
    role: str,
    parts: Iterable[ChatCompletionContentPartParam],
981
    mm_tracker: BaseMultiModalItemTracker,
982
983
    *,
    wrap_dicts: bool,
984
) -> list[ConversationMessage]:
985
    content = list[_ContentPart]()
986

987
    mm_parser = mm_tracker.create_parser()
988
989

    for part in parts:
990
        parse_res = _parse_chat_message_content_part(
991
992
993
994
            part,
            mm_parser,
            wrap_dicts=wrap_dicts,
        )
995
996
        if parse_res:
            content.append(parse_res)
997

998
    if wrap_dicts:
999
        # Parsing wraps images and texts as interleaved dictionaries
1000
        return [ConversationMessage(role=role,
1001
                                    content=content)]  # type: ignore
1002
    texts = cast(list[str], content)
1003
1004
1005
1006
1007
1008
1009
1010
1011
    text_prompt = "\n".join(texts)
    mm_placeholder_counts = mm_parser.mm_placeholder_counts()
    if mm_placeholder_counts:
        text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
                                                       text_prompt)
    return [ConversationMessage(role=role, content=text_prompt)]


def _parse_chat_message_content_part(
1012
1013
1014
1015
    part: ChatCompletionContentPartParam,
    mm_parser: BaseMultiModalContentParser,
    *,
    wrap_dicts: bool,
1016
) -> Optional[_ContentPart]:
1017
1018
1019
1020
1021
1022
1023
1024
    """Parses a single part of a conversation. If wrap_dicts is True,
    structured dictionary pieces for texts and images will be
    wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
    {"type": "image"}, respectively. Otherwise multimodal data will be
    handled by mm_parser, and texts will be returned as strings to be joined
    with multimodal placeholders.
    """
    if isinstance(part, str):  # Handle plain text parts
1025
        return part
1026
1027
1028
1029

    # Handle structured dictionary parts
    part_type, content = _parse_chat_message_content_mm_part(part)

1030
    # if part_type is text/refusal/image_url/audio_url/video_url/input_audio but
1031
1032
    # content is None, log a warning and skip
    if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and content is None:
1033
        logger.warning(
1034
1035
            "Skipping multimodal part '%s' (type: '%s') "
            "with empty / unparsable content.", part, part_type)
1036
1037
1038
        return None

    if part_type in ("text", "refusal"):
1039
1040
1041
1042
1043
        str_content = cast(str, content)
        if wrap_dicts:
            return {'type': 'text', 'text': str_content}
        else:
            return str_content
1044
1045

    if part_type == "image_url":
1046
1047
        str_content = cast(str, content)
        mm_parser.parse_image(str_content)
1048
        return {'type': 'image'} if wrap_dicts else None
1049
1050
1051
1052
    if part_type == "image_embeds":
        content = cast(Union[str, dict[str, str]], content)
        mm_parser.parse_image_embeds(content)
        return {'type': 'image'} if wrap_dicts else None
1053
    if part_type == "audio_url":
1054
1055
1056
1057
1058
        str_content = cast(str, content)
        mm_parser.parse_audio(str_content)
        return {'type': 'audio'} if wrap_dicts else None

    if part_type == "input_audio":
1059
        dict_content = cast(InputAudio, content)
1060
        mm_parser.parse_input_audio(dict_content)
1061
1062
        return {'type': 'audio'} if wrap_dicts else None

1063
    if part_type == "video_url":
1064
1065
        str_content = cast(str, content)
        mm_parser.parse_video(str_content)
1066
1067
        return {'type': 'video'} if wrap_dicts else None

1068
    raise NotImplementedError(f"Unknown part type: {part_type}")
1069
1070


1071
1072
1073
1074
1075
# No need to validate using Pydantic again
_AssistantParser = partial(cast, ChatCompletionAssistantMessageParam)
_ToolParser = partial(cast, ChatCompletionToolMessageParam)


1076
def _parse_chat_message_content(
1077
1078
    message: ChatCompletionMessageParam,
    mm_tracker: BaseMultiModalItemTracker,
1079
    content_format: _ChatTemplateContentFormat,
1080
) -> list[ConversationMessage]:
1081
1082
1083
1084
    role = message["role"]
    content = message.get("content")

    if content is None:
1085
1086
1087
1088
1089
1090
        content = []
    elif isinstance(content, str):
        content = [
            ChatCompletionContentPartTextParam(type="text", text=content)
        ]
    result = _parse_chat_message_content_parts(
1091
1092
        role,
        content,  # type: ignore
1093
        mm_tracker,
1094
        wrap_dicts=(content_format == "openai"),
1095
    )
1096

1097
1098
1099
1100
    for result_msg in result:
        if role == 'assistant':
            parsed_msg = _AssistantParser(message)

1101
1102
1103
1104
1105
            # The 'tool_calls' is not None check ensures compatibility.
            # It's needed only if downstream code doesn't strictly
            # follow the OpenAI spec.
            if ("tool_calls" in parsed_msg
                and parsed_msg["tool_calls"] is not None):
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
                result_msg["tool_calls"] = list(parsed_msg["tool_calls"])
        elif role == "tool":
            parsed_msg = _ToolParser(message)
            if "tool_call_id" in parsed_msg:
                result_msg["tool_call_id"] = parsed_msg["tool_call_id"]

        if "name" in message and isinstance(message["name"], str):
            result_msg["name"] = message["name"]

    return result

1117

1118
def _postprocess_messages(messages: list[ConversationMessage]) -> None:
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
    # per the Transformers docs & maintainers, tool call arguments in
    # assistant-role messages with tool_calls need to be dicts not JSON str -
    # this is how tool-use chat templates will expect them moving forwards
    # so, for messages that have tool_calls, parse the string (which we get
    # from openAI format) to dict
    for message in messages:
        if (message["role"] == "assistant" and "tool_calls" in message
                and isinstance(message["tool_calls"], list)):

            for item in message["tool_calls"]:
                item["function"]["arguments"] = json.loads(
                    item["function"]["arguments"])


1133
def parse_chat_messages(
1134
    messages: list[ChatCompletionMessageParam],
1135
    model_config: ModelConfig,
1136
    tokenizer: AnyTokenizer,
1137
    content_format: _ChatTemplateContentFormat,
1138
1139
) -> tuple[list[ConversationMessage], Optional[MultiModalDataDict]]:
    conversation: list[ConversationMessage] = []
1140
    mm_tracker = MultiModalItemTracker(model_config, tokenizer)
1141
1142

    for msg in messages:
1143
1144
1145
        sub_messages = _parse_chat_message_content(
            msg,
            mm_tracker,
1146
            content_format,
1147
        )
1148

1149
        conversation.extend(sub_messages)
1150

1151
1152
    _postprocess_messages(conversation)

1153
    return conversation, mm_tracker.all_mm_data()
1154
1155


1156
def parse_chat_messages_futures(
1157
    messages: list[ChatCompletionMessageParam],
1158
1159
    model_config: ModelConfig,
    tokenizer: AnyTokenizer,
1160
    content_format: _ChatTemplateContentFormat,
1161
1162
) -> tuple[list[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:
    conversation: list[ConversationMessage] = []
1163
1164
1165
    mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)

    for msg in messages:
1166
1167
1168
        sub_messages = _parse_chat_message_content(
            msg,
            mm_tracker,
1169
            content_format,
1170
        )
1171
1172
1173

        conversation.extend(sub_messages)

1174
1175
    _postprocess_messages(conversation)

1176
1177
1178
    return conversation, mm_tracker.all_mm_data()


1179
1180
def apply_hf_chat_template(
    tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
1181
    conversation: list[ConversationMessage],
1182
    chat_template: Optional[str],
1183
    tools: Optional[list[dict[str, Any]]],
1184
    *,
1185
    trust_remote_code: bool = False,
1186
1187
    tokenize: bool = False,  # Different from HF's default
    **kwargs: Any,
1188
) -> str:
1189
    hf_chat_template = resolve_hf_chat_template(
1190
1191
1192
1193
1194
        tokenizer,
        chat_template=chat_template,
        tools=tools,
        trust_remote_code=trust_remote_code,
    )
1195

1196
    if hf_chat_template is None:
1197
1198
1199
1200
1201
        raise ValueError(
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one.")

1202
1203
1204
1205
1206
1207
1208
1209
1210
    try:

        return tokenizer.apply_chat_template(
            conversation=conversation,  # type: ignore[arg-type]
            tools=tools,  # type: ignore[arg-type]
            chat_template=hf_chat_template,
            tokenize=tokenize,
            **kwargs,
        )
1211

1212
1213
1214
1215
1216
1217
1218
1219
1220
    # External library exceptions can sometimes occur despite the framework's
    # internal exception management capabilities.
    except Exception as e:

        # Log and report any library-related exceptions for further
        # investigation.
        logger.exception(
            "An error occurred in `transformers` while applying chat template")
        raise ValueError from e
1221

1222
1223
def apply_mistral_chat_template(
    tokenizer: MistralTokenizer,
1224
    messages: list[ChatCompletionMessageParam],
1225
1226
    chat_template: Optional[str],
    tools: Optional[list[dict[str, Any]]],
1227
    **kwargs: Any,
1228
) -> list[int]:
1229
1230
    from mistral_common.exceptions import MistralCommonException

1231
1232
1233
1234
1235
1236
    # The return value of resolve_mistral_chat_template is always None,
    # and we won't use it.
    resolve_mistral_chat_template(
        chat_template=chat_template,
        **kwargs,
    )
1237

1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
    try:
        return tokenizer.apply_chat_template(
            messages=messages,
            tools=tools,
            **kwargs,
        )
    # mistral-common uses assert statements to stop processing of input
    # if input does not comply with the expected format.
    # We convert those assertion errors to ValueErrors so they can be
    # are properly caught in the preprocessing_input step
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
    except (AssertionError, MistralCommonException) as e:
        raise ValueError from e

    # External library exceptions can sometimes occur despite the framework's
    # internal exception management capabilities.
    except Exception as e:

        # Log and report any library-related exceptions for further
        # investigation.
        logger.exception(
            "An error occurred in `mistral_common` while applying chat "
            "template")
1260
        raise ValueError from e