chat_utils.py 65.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
import asyncio
5
import inspect
6
import json
7
from abc import ABC, abstractmethod
8
from collections import Counter, defaultdict, deque
9
from collections.abc import Awaitable, Callable, Iterable
10
from functools import cached_property, lru_cache, partial
11
from pathlib import Path
12
from typing import TYPE_CHECKING, Any, Generic, Literal, TypeAlias, TypeVar, cast
13

14
15
16
import jinja2
import jinja2.ext
import jinja2.meta
17
import jinja2.nodes
18
19
import jinja2.parser
import jinja2.sandbox
20
import transformers.utils.chat_template_utils as hf_chat_utils
21
from openai.types.chat import (
22
23
24
25
26
    ChatCompletionAssistantMessageParam,
    ChatCompletionContentPartImageParam,
    ChatCompletionContentPartInputAudioParam,
    ChatCompletionContentPartRefusalParam,
    ChatCompletionContentPartTextParam,
27
    ChatCompletionFunctionToolParam,
28
29
30
31
32
33
    ChatCompletionMessageToolCallParam,
    ChatCompletionToolMessageParam,
)
from openai.types.chat import (
    ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam,
)
34
from openai.types.chat import (
35
36
37
    ChatCompletionMessageParam as OpenAIChatCompletionMessageParam,
)
from openai.types.chat.chat_completion_content_part_input_audio_param import InputAudio
38
from openai.types.responses import ResponseInputImageParam
39
from openai_harmony import Message as OpenAIHarmonyMessage
40
41
from PIL import Image
from pydantic import BaseModel, ConfigDict, TypeAdapter
42
43
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, ProcessorMixin

44
# pydantic needs the TypedDict from typing_extensions
45
from typing_extensions import Required, TypedDict
46

47
from vllm import envs
48
from vllm.config import ModelConfig
49
from vllm.logger import init_logger
50
from vllm.model_executor.models import SupportsMultiModal
51
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict
52
from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector
53
from vllm.tokenizers import TokenizerLike
54
from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path
55
from vllm.transformers_utils.processor import cached_get_processor
56
from vllm.utils import random_uuid
57
from vllm.utils.collection_utils import is_list_of
58
from vllm.utils.func_utils import supports_kw
59
60
61
62
from vllm.utils.import_utils import LazyLoader

if TYPE_CHECKING:
    import torch
63
64

    from vllm.tokenizers.mistral import MistralTokenizer
65
66
else:
    torch = LazyLoader("torch", globals(), "torch")
67
68
69

logger = init_logger(__name__)

70
71
72
73
74
75
76
77
78

class ChatTemplateResolutionError(ValueError):
    """Raised when chat template resolution fails.

    This is a subclass of ValueError for backward compatibility with
    existing exception handlers.
    """


79
80
81
82
83
84
MODALITY_PLACEHOLDERS_MAP = {
    "image": "<##IMAGE##>",
    "audio": "<##AUDIO##>",
    "video": "<##VIDEO##>",
}

85

86
87
88
89
90
91
92
93
94
95
96
97
98
99
class AudioURL(TypedDict, total=False):
    url: Required[str]
    """
    Either a URL of the audio or a data URL with base64 encoded audio data.
    """


class ChatCompletionContentPartAudioParam(TypedDict, total=False):
    audio_url: Required[AudioURL]

    type: Required[Literal["audio_url"]]
    """The type of the content part."""


100
class ChatCompletionContentPartImageEmbedsParam(TypedDict, total=False):
101
    image_embeds: str | dict[str, str] | None
102
103
104
105
106
107
108
    """
    The image embeddings. It can be either:
    - A single base64 string.
    - A dictionary where each value is a base64 string.
    """
    type: Required[Literal["image_embeds"]]
    """The type of the content part."""
109
    uuid: str | None
110
111
112
113
    """
    User-provided UUID of a media. User must guarantee that it is properly
    generated and unique for different medias.
    """
114
115


116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
    audio_embeds: str | dict[str, str] | None
    """
    The audio embeddings. It can be either:
    - A single base64 string representing a serialized torch tensor.
    - A dictionary where each value is a base64 string.
    """
    type: Required[Literal["audio_embeds"]]
    """The type of the content part."""
    uuid: str | None
    """
    User-provided UUID of a media. User must guarantee that it is properly
    generated and unique for different medias.
    """


132
133
134
135
136
137
138
139
140
141
142
143
144
145
class VideoURL(TypedDict, total=False):
    url: Required[str]
    """
    Either a URL of the video or a data URL with base64 encoded video data.
    """


class ChatCompletionContentPartVideoParam(TypedDict, total=False):
    video_url: Required[VideoURL]

    type: Required[Literal["video_url"]]
    """The type of the content part."""


146
147
148
149
class PILImage(BaseModel):
    """
    A PIL.Image.Image object.
    """
150

151
152
153
154
155
156
157
158
159
160
161
162
    image_pil: Image.Image
    model_config = ConfigDict(arbitrary_types_allowed=True)


class CustomChatCompletionContentPILImageParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a PIL image.

    Example:
    {
        "image_pil": ImageAsset('cherry_blossom').pil_image
    }
    """
163

164
165
    image_pil: PILImage | None
    uuid: str | None
166
167
168
169
    """
    User-provided UUID of a media. User must guarantee that it is properly
    generated and unique for different medias.
    """
170
171


172
173
174
class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain image_url.
    This is supported by OpenAI API, although it is not documented.
175

176
177
178
179
180
    Example:
    {
        "image_url": "https://example.com/image.jpg"
    }
    """
181

182
183
    image_url: str | None
    uuid: str | None
184
185
186
187
    """
    User-provided UUID of a media. User must guarantee that it is properly
    generated and unique for different medias.
    """
188
189
190
191


class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain audio_url.
192

193
194
195
196
197
    Example:
    {
        "audio_url": "https://example.com/audio.mp3"
    }
    """
198

199
    audio_url: str | None
200
201


202
203
204
205
206
207
208
209
class CustomChatCompletionContentSimpleVideoParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain audio_url.

    Example:
    {
        "video_url": "https://example.com/video.mp4"
    }
    """
210

211
212
    video_url: str | None
    uuid: str | None
213
214
215
216
    """
    User-provided UUID of a media. User must guarantee that it is properly
    generated and unique for different medias.
    """
217
218


Julien Denize's avatar
Julien Denize committed
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
class CustomThinkCompletionContentParam(TypedDict, total=False):
    """A Think Completion Content Param that accepts a plain text and a boolean.

    Example:
    {
        "thinking": "I am thinking about the answer",
        "closed": True,
        "type": "thinking"
    }
    """

    thinking: Required[str]
    """The thinking content."""

    closed: bool
    """Whether the thinking is closed."""

    type: Required[Literal["thinking"]]
    """The thinking type."""


240
241
242
243
244
245
246
247
248
ChatCompletionContentPartParam: TypeAlias = (
    OpenAIChatCompletionContentPartParam
    | ChatCompletionContentPartAudioParam
    | ChatCompletionContentPartInputAudioParam
    | ChatCompletionContentPartVideoParam
    | ChatCompletionContentPartRefusalParam
    | CustomChatCompletionContentPILImageParam
    | CustomChatCompletionContentSimpleImageParam
    | ChatCompletionContentPartImageEmbedsParam
249
    | ChatCompletionContentPartAudioEmbedsParam
250
251
252
253
254
    | CustomChatCompletionContentSimpleAudioParam
    | CustomChatCompletionContentSimpleVideoParam
    | str
    | CustomThinkCompletionContentParam
)
255
256
257
258


class CustomChatCompletionMessageParam(TypedDict, total=False):
    """Enables custom roles in the Chat Completion API."""
259

260
261
262
    role: Required[str]
    """The role of the message's author."""

263
    content: str | list[ChatCompletionContentPartParam]
264
265
266
267
268
269
270
271
272
    """The contents of the message."""

    name: str
    """An optional name for the participant.

    Provides the model information to differentiate between participants of the
    same role.
    """

273
    tool_call_id: str | None
274
275
    """Tool call that this message is responding to."""

276
    tool_calls: Iterable[ChatCompletionMessageToolCallParam] | None
277
278
    """The tool calls generated by the model, such as function calls."""

279
280
281
    reasoning: str | None
    """The reasoning content for interleaved thinking."""

282
283
284
    tools: list[ChatCompletionFunctionToolParam] | None
    """The tools for developer role."""

285

286
287
288
289
290
ChatCompletionMessageParam: TypeAlias = (
    OpenAIChatCompletionMessageParam
    | CustomChatCompletionMessageParam
    | OpenAIHarmonyMessage
)
291
292


293
# TODO: Make fields ReadOnly once mypy supports it
294
295
296
297
class ConversationMessage(TypedDict, total=False):
    role: Required[str]
    """The role of the message's author."""

298
    content: str | None | list[dict[str, str]]
299
300
    """The contents of the message"""

301
    tool_call_id: str | None
302
303
    """Tool call that this message is responding to."""

304
    name: str | None
305
306
    """The name of the function to call"""

307
    tool_calls: Iterable[ChatCompletionMessageToolCallParam] | None
308
    """The tool calls generated by the model, such as function calls."""
309

310
311
312
313
314
315
    reasoning: str | None
    """The reasoning content for interleaved thinking."""

    reasoning_content: str | None
    """Deprecated: The reasoning content for interleaved thinking."""

316
317
318
    tools: list[ChatCompletionFunctionToolParam] | None
    """The tools for developer role."""

319

320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
# Passed in by user
ChatTemplateContentFormatOption = Literal["auto", "string", "openai"]

# Used internally
_ChatTemplateContentFormat = Literal["string", "openai"]


def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool:
    if isinstance(node, jinja2.nodes.Name):
        return node.ctx == "load" and node.name == varname

    return False


def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool:
    if isinstance(node, jinja2.nodes.Getitem):
336
337
338
339
340
        return (
            _is_var_access(node.node, varname)
            and isinstance(node.arg, jinja2.nodes.Const)
            and node.arg.value == key
        )
341
342
343
344
345
346
347
348
349
350

    if isinstance(node, jinja2.nodes.Getattr):
        return _is_var_access(node.node, varname) and node.attr == key

    return False


def _is_var_or_elems_access(
    node: jinja2.nodes.Node,
    varname: str,
351
    key: str | None = None,
352
353
) -> bool:
    if isinstance(node, jinja2.nodes.Filter):
354
        return node.node is not None and _is_var_or_elems_access(
355
356
            node.node, varname, key
        )
357
358
359
    if isinstance(node, jinja2.nodes.Test):
        return _is_var_or_elems_access(node.node, varname, key)

360
    if isinstance(node, jinja2.nodes.Getitem) and isinstance(
361
362
        node.arg, jinja2.nodes.Slice
    ):
363
364
        return _is_var_or_elems_access(node.node, varname, key)

365
    return _is_attr_access(node, varname, key) if key else _is_var_access(node, varname)
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393


def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str):
    # Global variable that is implicitly defined at the root
    yield root, varname

    # Iterative BFS
    related_varnames = deque([varname])
    while related_varnames:
        related_varname = related_varnames.popleft()

        for assign_ast in root.find_all(jinja2.nodes.Assign):
            lhs = assign_ast.target
            rhs = assign_ast.node

            if _is_var_or_elems_access(rhs, related_varname):
                assert isinstance(lhs, jinja2.nodes.Name)
                yield assign_ast, lhs.name

                # Avoid infinite looping for self-assignment
                if lhs.name != related_varname:
                    related_varnames.append(lhs.name)


# NOTE: The proper way to handle this is to build a CFG so that we can handle
# the scope in which each variable is defined, but that is too complicated
def _iter_nodes_assign_messages_item(root: jinja2.nodes.Node):
    messages_varnames = [
394
        varname for _, varname in _iter_nodes_assign_var_or_elems(root, "messages")
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
    ]

    # Search for {%- for message in messages -%} loops
    for loop_ast in root.find_all(jinja2.nodes.For):
        loop_iter = loop_ast.iter
        loop_target = loop_ast.target

        for varname in messages_varnames:
            if _is_var_or_elems_access(loop_iter, varname):
                assert isinstance(loop_target, jinja2.nodes.Name)
                yield loop_ast, loop_target.name
                break


def _iter_nodes_assign_content_item(root: jinja2.nodes.Node):
    message_varnames = [
        varname for _, varname in _iter_nodes_assign_messages_item(root)
    ]

    # Search for {%- for content in message['content'] -%} loops
    for loop_ast in root.find_all(jinja2.nodes.For):
        loop_iter = loop_ast.iter
        loop_target = loop_ast.target

        for varname in message_varnames:
            if _is_var_or_elems_access(loop_iter, varname, "content"):
                assert isinstance(loop_target, jinja2.nodes.Name)
                yield loop_ast, loop_target.name
                break


426
def _try_extract_ast(chat_template: str) -> jinja2.nodes.Template | None:
427
428
429
430
431
432
433
434
    try:
        jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template)
        return jinja_compiled.environment.parse(chat_template)
    except Exception:
        logger.exception("Error when compiling Jinja template")
        return None


435
@lru_cache(maxsize=32)
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
def _detect_content_format(
    chat_template: str,
    *,
    default: _ChatTemplateContentFormat,
) -> _ChatTemplateContentFormat:
    jinja_ast = _try_extract_ast(chat_template)
    if jinja_ast is None:
        return default

    try:
        next(_iter_nodes_assign_content_item(jinja_ast))
    except StopIteration:
        return "string"
    except Exception:
        logger.exception("Error when parsing AST of Jinja template")
        return default
    else:
        return "openai"


456
def resolve_mistral_chat_template(
457
    chat_template: str | None,
458
    **kwargs: Any,
459
) -> str | None:
460
461
462
463
    if chat_template is not None or kwargs.get("chat_template_kwargs") is not None:
        raise ValueError(
            "'chat_template' or 'chat_template_kwargs' cannot be overridden "
            "for mistral tokenizer."
464
        )
465

466
467
    return None

468

469
_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]()
470
471
472
473
474
475
476
477
478
"""
Used in `_try_get_processor_chat_template` to avoid calling
`cached_get_processor` again if the processor fails to be loaded.

This is needed because `lru_cache` does not cache when an exception happens.
"""


def _try_get_processor_chat_template(
479
    tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
480
    model_config: ModelConfig,
481
) -> str | None:
482
    cache_key = (tokenizer.name_or_path, model_config.trust_remote_code)
483
484
485
486
487
488
489
490
491
492
493
    if cache_key in _PROCESSOR_CHAT_TEMPLATES:
        return _PROCESSOR_CHAT_TEMPLATES[cache_key]

    try:
        processor = cached_get_processor(
            tokenizer.name_or_path,
            processor_cls=(
                PreTrainedTokenizer,
                PreTrainedTokenizerFast,
                ProcessorMixin,
            ),
494
            trust_remote_code=model_config.trust_remote_code,
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
        )
        if (
            isinstance(processor, ProcessorMixin)
            and hasattr(processor, "chat_template")
            and (chat_template := processor.chat_template) is not None
        ):
            _PROCESSOR_CHAT_TEMPLATES[cache_key] = chat_template
            return chat_template
    except Exception:
        logger.debug(
            "Failed to load AutoProcessor chat template for %s",
            tokenizer.name_or_path,
            exc_info=True,
        )

    _PROCESSOR_CHAT_TEMPLATES[cache_key] = None
    return None


514
def resolve_hf_chat_template(
515
516
517
    tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
    chat_template: str | None,
    tools: list[dict[str, Any]] | None,
518
519
    *,
    model_config: ModelConfig,
520
) -> str | None:
521
522
523
524
525
526
    # 1st priority: The given chat template
    if chat_template is not None:
        return chat_template

    # 2nd priority: AutoProcessor chat template, unless tool calling is enabled
    if tools is None:
527
        chat_template = _try_get_processor_chat_template(tokenizer, model_config)
528
529
        if chat_template is not None:
            return chat_template
530
531
532
533
534

    # 3rd priority: AutoTokenizer chat template
    try:
        return tokenizer.get_chat_template(chat_template, tools=tools)
    except Exception:
535
536
537
538
539
        logger.debug(
            "Failed to load AutoTokenizer chat template for %s",
            tokenizer.name_or_path,
            exc_info=True,
        )
540

541
    # 4th priority: Predefined fallbacks
542
543
    path = get_chat_template_fallback_path(
        model_type=model_config.hf_config.model_type,
544
        tokenizer_name_or_path=model_config.tokenizer,
545
546
    )
    if path is not None:
547
        logger.info_once(
548
549
550
551
            "Loading chat template fallback for %s as there isn't one "
            "defined on HF Hub.",
            tokenizer.name_or_path,
        )
552
553
        chat_template = load_chat_template(path)
    else:
554
        logger.debug_once(
555
556
            "There is no chat template fallback for %s", tokenizer.name_or_path
        )
557
558

    return chat_template
559
560


561
def _resolve_chat_template_content_format(
562
563
    chat_template: str | None,
    tools: list[dict[str, Any]] | None,
564
    tokenizer: TokenizerLike | None,
565
    *,
566
    model_config: ModelConfig,
567
568
) -> _ChatTemplateContentFormat:
    if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)):
569
        hf_chat_template = resolve_hf_chat_template(
570
571
572
            tokenizer,
            chat_template=chat_template,
            tools=tools,
573
            model_config=model_config,
574
        )
575
    else:
576
577
        hf_chat_template = None

578
579
580
581
582
    jinja_text = (
        hf_chat_template
        if isinstance(hf_chat_template, str)
        else load_chat_template(chat_template, is_literal=True)
    )
583

584
585
586
587
588
    detected_format = (
        "string"
        if jinja_text is None
        else _detect_content_format(jinja_text, default="string")
    )
589

590
    return detected_format
591
592
593


@lru_cache
594
def _log_chat_template_content_format(
595
    chat_template: str | None,
596
    given_format: ChatTemplateContentFormatOption,
597
598
    detected_format: ChatTemplateContentFormatOption,
):
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
    logger.info(
        "Detected the chat template content format to be '%s'. "
        "You can set `--chat-template-content-format` to override this.",
        detected_format,
    )

    if given_format != "auto" and given_format != detected_format:
        logger.warning(
            "You specified `--chat-template-content-format %s` "
            "which is different from the detected format '%s'. "
            "If our automatic detection is incorrect, please consider "
            "opening a GitHub issue so that we can improve it: "
            "https://github.com/vllm-project/vllm/issues/new/choose",
            given_format,
            detected_format,
        )

616
617

def resolve_chat_template_content_format(
618
619
    chat_template: str | None,
    tools: list[dict[str, Any]] | None,
620
    given_format: ChatTemplateContentFormatOption,
621
    tokenizer: TokenizerLike | None,
622
    *,
623
    model_config: ModelConfig,
624
) -> _ChatTemplateContentFormat:
625
626
627
    if given_format != "auto":
        return given_format

628
629
630
631
    detected_format = _resolve_chat_template_content_format(
        chat_template,
        tools,
        tokenizer,
632
        model_config=model_config,
633
634
635
636
637
638
639
640
    )

    _log_chat_template_content_format(
        chat_template,
        given_format=given_format,
        detected_format=detected_format,
    )

641
    return detected_format
642

643

644
ModalityStr = Literal["image", "audio", "video", "image_embeds", "audio_embeds"]
645
646
647
_T = TypeVar("_T")


648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
def _extract_embeds(tensors: list[torch.Tensor]):
    if len(tensors) == 0:
        return tensors

    if len(tensors) == 1:
        tensors[0]._is_single_item = True  # type: ignore
        return tensors[0]  # To keep backwards compatibility for single item input

    first_shape = tensors[0].shape
    if all(t.shape == first_shape for t in tensors):
        return torch.stack(tensors)

    return tensors


def _get_embeds_data(items_by_modality: dict[str, list[Any]], modality: str):
    embeds_key = f"{modality}_embeds"
    embeds = items_by_modality[embeds_key]

    if len(embeds) == 0:
        return embeds
    if is_list_of(embeds, torch.Tensor):
        return _extract_embeds(embeds)
    if is_list_of(embeds, dict):
        if not embeds:
            return {}

        first_keys = set(embeds[0].keys())
        if any(set(item.keys()) != first_keys for item in embeds[1:]):
            raise ValueError(
                "All dictionaries in the list of embeddings must have the same keys."
            )

        return {k: _extract_embeds([item[k] for item in embeds]) for k in first_keys}

    return embeds


686
class BaseMultiModalItemTracker(ABC, Generic[_T]):
687
688
689
690
691
692
    """
    Tracks multi-modal items in a given request and ensures that the number
    of multi-modal items in a given request does not exceed the configured
    maximum per prompt.
    """

693
    def __init__(self, model_config: ModelConfig):
694
695
        super().__init__()

696
        self._model_config = model_config
697

698
699
        self._items_by_modality = defaultdict[str, list[_T | None]](list)
        self._uuids_by_modality = defaultdict[str, list[str | None]](list)
700

701
    @property
702
703
    def model_config(self) -> ModelConfig:
        return self._model_config
704

705
    @cached_property
706
    def model_cls(self) -> type[SupportsMultiModal]:
707
        from vllm.model_executor.model_loader import get_model_cls
708

709
        model_cls = get_model_cls(self.model_config)
710
        return cast(type[SupportsMultiModal], model_cls)
711

712
713
    @property
    def allowed_local_media_path(self):
714
        return self._model_config.allowed_local_media_path
715

716
717
    @property
    def allowed_media_domains(self):
718
        return self._model_config.allowed_media_domains
719

720
721
722
723
    @property
    def mm_registry(self):
        return MULTIMODAL_REGISTRY

724
725
    @cached_property
    def mm_processor(self):
726
        return self.mm_registry.create_processor(self.model_config)
727

728
    def add(
729
730
        self,
        modality: ModalityStr,
731
732
733
        item: _T | None,
        uuid: str | None = None,
    ) -> str | None:
734
735
736
        """
        Add a multi-modal item to the current prompt and returns the
        placeholder string to use, if any.
737
738

        An optional uuid can be added which serves as a unique identifier of the
739
        media.
740
        """
741
        input_modality = modality.replace("_embeds", "")
742
        num_items = len(self._items_by_modality[modality]) + 1
743

744
        self.mm_processor.validate_num_items(input_modality, num_items)
745

746
        self._items_by_modality[modality].append(item)
747
        self._uuids_by_modality[modality].append(uuid)
748

749
        return self.model_cls.get_placeholder_str(modality, num_items)
750

751
    def all_mm_uuids(self) -> MultiModalUUIDDict | None:
752
753
        if not self._items_by_modality:
            return None
754

755
756
        uuids_by_modality = dict(self._uuids_by_modality)
        if "image" in uuids_by_modality and "image_embeds" in uuids_by_modality:
757
            raise ValueError("Mixing raw image and embedding inputs is not allowed")
758
759
        if "audio" in uuids_by_modality and "audio_embeds" in uuids_by_modality:
            raise ValueError("Mixing raw audio and embedding inputs is not allowed")
760

761
        mm_uuids = {}
762
763
764
765
        if "image_embeds" in uuids_by_modality:
            mm_uuids["image"] = uuids_by_modality["image_embeds"]
        if "image" in uuids_by_modality:
            mm_uuids["image"] = uuids_by_modality["image"]  # UUIDs of images
766
767
        if "audio_embeds" in uuids_by_modality:
            mm_uuids["audio"] = uuids_by_modality["audio_embeds"]
768
769
770
771
        if "audio" in uuids_by_modality:
            mm_uuids["audio"] = uuids_by_modality["audio"]  # UUIDs of audios
        if "video" in uuids_by_modality:
            mm_uuids["video"] = uuids_by_modality["video"]  # UUIDs of videos
772

773
774
        return mm_uuids

775
776
777
778
779
    @abstractmethod
    def create_parser(self) -> "BaseMultiModalContentParser":
        raise NotImplementedError


780
class MultiModalItemTracker(BaseMultiModalItemTracker[object]):
781
    def all_mm_data(self) -> MultiModalDataDict | None:
782
783
        if not self._items_by_modality:
            return None
784

785
786
        items_by_modality = dict(self._items_by_modality)
        if "image" in items_by_modality and "image_embeds" in items_by_modality:
787
            raise ValueError("Mixing raw image and embedding inputs is not allowed")
788
789
        if "audio" in items_by_modality and "audio_embeds" in items_by_modality:
            raise ValueError("Mixing raw audio and embedding inputs is not allowed")
790

791
        mm_inputs = {}
792
        if "image_embeds" in items_by_modality:
793
            mm_inputs["image"] = _get_embeds_data(items_by_modality, "image")
794
        if "image" in items_by_modality:
795
            mm_inputs["image"] = items_by_modality["image"]  # A list of images
796
        if "audio_embeds" in items_by_modality:
797
            mm_inputs["audio"] = _get_embeds_data(items_by_modality, "audio")
798
        if "audio" in items_by_modality:
799
            mm_inputs["audio"] = items_by_modality["audio"]  # A list of audios
800
        if "video" in items_by_modality:
801
            mm_inputs["video"] = items_by_modality["video"]  # A list of videos
802

803
        return mm_inputs
804
805
806
807
808

    def create_parser(self) -> "BaseMultiModalContentParser":
        return MultiModalContentParser(self)


809
class AsyncMultiModalItemTracker(BaseMultiModalItemTracker[Awaitable[object]]):
810
    async def all_mm_data(self) -> MultiModalDataDict | None:
811
812
        if not self._items_by_modality:
            return None
813

814
815
816
817
818
819
820
821
        coros_by_modality = {
            modality: [item or asyncio.sleep(0) for item in items]
            for modality, items in self._items_by_modality.items()
        }
        items_by_modality: dict[str, list[object | None]] = {
            modality: await asyncio.gather(*coros)
            for modality, coros in coros_by_modality.items()
        }
822
        if "image" in items_by_modality and "image_embeds" in items_by_modality:
823
            raise ValueError("Mixing raw image and embedding inputs is not allowed")
824
825
        if "audio" in items_by_modality and "audio_embeds" in items_by_modality:
            raise ValueError("Mixing raw audio and embedding inputs is not allowed")
826

827
        mm_inputs = {}
828
        if "image_embeds" in items_by_modality:
829
            mm_inputs["image"] = _get_embeds_data(items_by_modality, "image")
830
        if "image" in items_by_modality:
831
            mm_inputs["image"] = items_by_modality["image"]  # A list of images
832
        if "audio_embeds" in items_by_modality:
833
            mm_inputs["audio"] = _get_embeds_data(items_by_modality, "audio")
834
        if "audio" in items_by_modality:
835
            mm_inputs["audio"] = items_by_modality["audio"]  # A list of audios
836
        if "video" in items_by_modality:
837
            mm_inputs["video"] = items_by_modality["video"]  # A list of videos
838

839
        return mm_inputs
840
841
842
843
844
845
846
847
848

    def create_parser(self) -> "BaseMultiModalContentParser":
        return AsyncMultiModalContentParser(self)


class BaseMultiModalContentParser(ABC):
    def __init__(self) -> None:
        super().__init__()

849
        # stores model placeholders list with corresponding
850
851
852
853
854
855
856
        # general MM placeholder:
        # {
        #   "<##IMAGE##>": ["<image>", "<image>", "<image>"],
        #   "<##AUDIO##>": ["<audio>", "<audio>"]
        # }
        self._placeholder_storage: dict[str, list] = defaultdict(list)

857
    def _add_placeholder(self, modality: ModalityStr, placeholder: str | None):
858
        mod_placeholder = MODALITY_PLACEHOLDERS_MAP[modality]
859
        if placeholder:
860
            self._placeholder_storage[mod_placeholder].append(placeholder)
861

862
863
    def mm_placeholder_storage(self) -> dict[str, list]:
        return dict(self._placeholder_storage)
864
865

    @abstractmethod
866
    def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
867
868
        raise NotImplementedError

869
    @abstractmethod
870
    def parse_image_embeds(
871
        self,
872
873
        image_embeds: str | dict[str, str] | None,
        uuid: str | None = None,
874
    ) -> None:
875
876
        raise NotImplementedError

877
    @abstractmethod
878
    def parse_image_pil(
879
        self, image_pil: Image.Image | None, uuid: str | None = None
880
    ) -> None:
881
882
        raise NotImplementedError

883
    @abstractmethod
884
    def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
885
886
        raise NotImplementedError

887
    @abstractmethod
888
    def parse_input_audio(
889
        self, input_audio: InputAudio | None, uuid: str | None = None
890
    ) -> None:
891
892
        raise NotImplementedError

893
894
895
896
897
898
899
900
    @abstractmethod
    def parse_audio_embeds(
        self,
        audio_embeds: str | dict[str, str] | None,
        uuid: str | None = None,
    ) -> None:
        raise NotImplementedError

901
    @abstractmethod
902
    def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
903
904
        raise NotImplementedError

905
906
907
908
909
910

class MultiModalContentParser(BaseMultiModalContentParser):
    def __init__(self, tracker: MultiModalItemTracker) -> None:
        super().__init__()

        self._tracker = tracker
911
912
913
        multimodal_config = self._tracker.model_config.multimodal_config
        media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)

914
915
        self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
            envs.VLLM_MEDIA_CONNECTOR,
916
            media_io_kwargs=media_io_kwargs,
917
            allowed_local_media_path=tracker.allowed_local_media_path,
918
            allowed_media_domains=tracker.allowed_media_domains,
919
920
        )

921
922
    @property
    def model_config(self) -> ModelConfig:
923
        return self._tracker.model_config
924

925
    def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
926
        image = self._connector.fetch_image(image_url) if image_url else None
927

928
        placeholder = self._tracker.add("image", image, uuid)
929
        self._add_placeholder("image", placeholder)
930

931
    def parse_image_embeds(
932
        self,
933
934
        image_embeds: str | dict[str, str] | None,
        uuid: str | None = None,
935
    ) -> None:
936
937
938
939
940
941
        mm_config = self.model_config.get_multimodal_config()
        if not mm_config.enable_mm_embeds:
            raise ValueError(
                "You must set `--enable-mm-embeds` to input `image_embeds`"
            )

942
943
944
945
946
        if isinstance(image_embeds, dict):
            embeds = {
                k: self._connector.fetch_image_embedding(v)
                for k, v in image_embeds.items()
            }
947
            placeholder = self._tracker.add("image_embeds", embeds, uuid)
948
949
950

        if isinstance(image_embeds, str):
            embedding = self._connector.fetch_image_embedding(image_embeds)
951
            placeholder = self._tracker.add("image_embeds", embedding, uuid)
952

953
954
955
        if image_embeds is None:
            placeholder = self._tracker.add("image_embeds", None, uuid)

956
        self._add_placeholder("image", placeholder)
957

958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
    def parse_audio_embeds(
        self,
        audio_embeds: str | dict[str, str] | None,
        uuid: str | None = None,
    ) -> None:
        mm_config = self.model_config.get_multimodal_config()
        if not mm_config.enable_mm_embeds:
            raise ValueError(
                "You must set `--enable-mm-embeds` to input `audio_embeds`"
            )

        if isinstance(audio_embeds, dict):
            embeds = {
                k: self._connector.fetch_audio_embedding(v)
                for k, v in audio_embeds.items()
            }
            placeholder = self._tracker.add("audio_embeds", embeds, uuid)
        elif isinstance(audio_embeds, str):
            embedding = self._connector.fetch_audio_embedding(audio_embeds)
            placeholder = self._tracker.add("audio_embeds", embedding, uuid)
        else:
            placeholder = self._tracker.add("audio_embeds", None, uuid)

        self._add_placeholder("audio", placeholder)

983
    def parse_image_pil(
984
        self, image_pil: Image.Image | None, uuid: str | None = None
985
986
    ) -> None:
        placeholder = self._tracker.add("image", image_pil, uuid)
987
        self._add_placeholder("image", placeholder)
988

989
    def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
990
        audio = self._connector.fetch_audio(audio_url) if audio_url else None
991

992
        placeholder = self._tracker.add("audio", audio, uuid)
993
        self._add_placeholder("audio", placeholder)
994

995
    def parse_input_audio(
996
        self, input_audio: InputAudio | None, uuid: str | None = None
997
    ) -> None:
998
999
1000
1001
1002
1003
1004
1005
1006
1007
        if input_audio:
            audio_data = input_audio.get("data", "")
            audio_format = input_audio.get("format", "")
            if audio_data:
                audio_url = f"data:audio/{audio_format};base64,{audio_data}"
            else:
                # If a UUID is provided, audio data may be empty.
                audio_url = None
        else:
            audio_url = None
1008

1009
        return self.parse_audio(audio_url, uuid)
1010

1011
    def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
1012
        video = self._connector.fetch_video(video_url=video_url) if video_url else None
1013

1014
        placeholder = self._tracker.add("video", video, uuid)
1015
        self._add_placeholder("video", placeholder)
1016

1017
1018
1019
1020
1021
1022

class AsyncMultiModalContentParser(BaseMultiModalContentParser):
    def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:
        super().__init__()

        self._tracker = tracker
1023
1024
        multimodal_config = self._tracker.model_config.multimodal_config
        media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
1025
1026
        self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
            envs.VLLM_MEDIA_CONNECTOR,
1027
            media_io_kwargs=media_io_kwargs,
1028
            allowed_local_media_path=tracker.allowed_local_media_path,
1029
            allowed_media_domains=tracker.allowed_media_domains,
1030
        )
1031

1032
1033
    @property
    def model_config(self) -> ModelConfig:
1034
        return self._tracker.model_config
1035

1036
    def parse_image(self, image_url: str | None, uuid: str | None = None) -> None:
1037
        image_coro = self._connector.fetch_image_async(image_url) if image_url else None
1038

1039
        placeholder = self._tracker.add("image", image_coro, uuid)
1040
        self._add_placeholder("image", placeholder)
1041

1042
    def parse_image_embeds(
1043
        self,
1044
1045
        image_embeds: str | dict[str, str] | None,
        uuid: str | None = None,
1046
    ) -> None:
1047
1048
1049
1050
1051
1052
        mm_config = self.model_config.get_multimodal_config()
        if not mm_config.enable_mm_embeds:
            raise ValueError(
                "You must set `--enable-mm-embeds` to input `image_embeds`"
            )

1053
        future: asyncio.Future[str | dict[str, str] | None] = asyncio.Future()
1054
1055
1056
1057
1058
1059
1060
1061
1062

        if isinstance(image_embeds, dict):
            embeds = {
                k: self._connector.fetch_image_embedding(v)
                for k, v in image_embeds.items()
            }
            future.set_result(embeds)

        if isinstance(image_embeds, str):
1063
            embedding = self._connector.fetch_image_embedding(image_embeds)
1064
1065
            future.set_result(embedding)

1066
1067
1068
        if image_embeds is None:
            future.set_result(None)

1069
        placeholder = self._tracker.add("image_embeds", future, uuid)
1070
        self._add_placeholder("image", placeholder)
1071

1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
    def parse_audio_embeds(
        self,
        audio_embeds: str | dict[str, str] | None,
        uuid: str | None = None,
    ) -> None:
        mm_config = self.model_config.get_multimodal_config()
        if not mm_config.enable_mm_embeds:
            raise ValueError(
                "You must set `--enable-mm-embeds` to input `audio_embeds`"
            )

        logger.info(
            "🎵 Parsing audio_embeds: type=%s, uuid=%s, is_dict=%s, "
            "is_str=%s, is_none=%s",
            type(audio_embeds).__name__,
            uuid,
            isinstance(audio_embeds, dict),
            isinstance(audio_embeds, str),
            audio_embeds is None,
        )

        future: asyncio.Future[str | dict[str, str] | None] = asyncio.Future()

        if isinstance(audio_embeds, dict):
            logger.info(
                "🎵 Processing dict audio_embeds with %d entries",
                len(audio_embeds),
            )
            embeds = {
                k: self._connector.fetch_audio_embedding(v)
                for k, v in audio_embeds.items()
            }
            future.set_result(embeds)
            logger.info(
                "🎵 Successfully loaded %d audio embeddings from dict",
                len(embeds),
            )

        if isinstance(audio_embeds, str):
            base64_size = len(audio_embeds)
            logger.info(
                "🎵 Processing base64 audio_embeds: %d chars (%.2f KB)",
                base64_size,
                base64_size / 1024,
            )
            embedding = self._connector.fetch_audio_embedding(audio_embeds)
            future.set_result(embedding)
            logger.info(
                "🎵 Successfully loaded audio embedding tensor: shape=%s, dtype=%s",
                embedding.shape,
                embedding.dtype,
            )

        if audio_embeds is None:
            logger.info("🎵 Audio embeds is None (UUID-only reference)")
            future.set_result(None)

        placeholder = self._tracker.add("audio_embeds", future, uuid)
        self._add_placeholder("audio", placeholder)
        logger.info("🎵 Added audio_embeds placeholder with uuid=%s", uuid)

1133
    def parse_image_pil(
1134
        self, image_pil: Image.Image | None, uuid: str | None = None
1135
    ) -> None:
1136
        future: asyncio.Future[Image.Image | None] = asyncio.Future()
1137
1138
1139
1140
        if image_pil:
            future.set_result(image_pil)
        else:
            future.set_result(None)
1141

1142
        placeholder = self._tracker.add("image", future, uuid)
1143
        self._add_placeholder("image", placeholder)
1144

1145
    def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None:
1146
        audio_coro = self._connector.fetch_audio_async(audio_url) if audio_url else None
1147

1148
        placeholder = self._tracker.add("audio", audio_coro, uuid)
1149
        self._add_placeholder("audio", placeholder)
1150

1151
    def parse_input_audio(
1152
        self, input_audio: InputAudio | None, uuid: str | None = None
1153
    ) -> None:
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
        if input_audio:
            audio_data = input_audio.get("data", "")
            audio_format = input_audio.get("format", "")
            if audio_data:
                audio_url = f"data:audio/{audio_format};base64,{audio_data}"
            else:
                # If a UUID is provided, audio data may be empty.
                audio_url = None
        else:
            audio_url = None
1164

1165
        return self.parse_audio(audio_url, uuid)
1166

1167
    def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
1168
1169
1170
1171
1172
        video = (
            self._connector.fetch_video_async(video_url=video_url)
            if video_url
            else None
        )
1173

1174
        placeholder = self._tracker.add("video", video, uuid)
1175
        self._add_placeholder("video", placeholder)
1176

1177

1178
def validate_chat_template(chat_template: Path | str | None):
1179
1180
1181
1182
1183
    """Raises if the provided chat template appears invalid."""
    if chat_template is None:
        return

    elif isinstance(chat_template, Path) and not chat_template.exists():
1184
        raise FileNotFoundError("the supplied chat template path doesn't exist")
1185
1186
1187

    elif isinstance(chat_template, str):
        JINJA_CHARS = "{}\n"
1188
1189
1190
1191
        if (
            not any(c in chat_template for c in JINJA_CHARS)
            and not Path(chat_template).exists()
        ):
1192
1193
1194
            # Try to find the template in the built-in templates directory
            from vllm.transformers_utils.chat_templates.registry import (
                CHAT_TEMPLATES_DIR,
1195
            )
1196

1197
1198
1199
1200
1201
1202
1203
1204
            builtin_template_path = CHAT_TEMPLATES_DIR / chat_template
            if not builtin_template_path.exists():
                raise ValueError(
                    f"The supplied chat template string ({chat_template}) "
                    f"appears path-like, but doesn't exist! "
                    f"Tried: {chat_template} and {builtin_template_path}"
                )

1205
    else:
1206
        raise TypeError(f"{type(chat_template)} is not a valid chat template type")
1207
1208


1209
def _load_chat_template(
1210
    chat_template: Path | str | None,
1211
1212
    *,
    is_literal: bool = False,
1213
) -> str | None:
1214
1215
    if chat_template is None:
        return None
1216
1217
1218

    if is_literal:
        if isinstance(chat_template, Path):
1219
1220
1221
            raise TypeError(
                "chat_template is expected to be read directly from its value"
            )
1222

1223
        return chat_template
1224

1225
    try:
1226
        with open(chat_template) as f:
1227
            return f.read()
1228
    except OSError as e:
1229
1230
1231
        if isinstance(chat_template, Path):
            raise

1232
1233
        JINJA_CHARS = "{}\n"
        if not any(c in chat_template for c in JINJA_CHARS):
1234
1235
1236
            # Try to load from the built-in templates directory
            from vllm.transformers_utils.chat_templates.registry import (
                CHAT_TEMPLATES_DIR,
1237
            )
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250

            builtin_template_path = CHAT_TEMPLATES_DIR / chat_template
            try:
                with open(builtin_template_path) as f:
                    return f.read()
            except OSError:
                msg = (
                    f"The supplied chat template ({chat_template}) "
                    f"looks like a file path, but it failed to be opened. "
                    f"Tried: {chat_template} and {builtin_template_path}. "
                    f"Reason: {e}"
                )
                raise ValueError(msg) from e
1251

1252
1253
        # If opening a file fails, set chat template to be args to
        # ensure we decode so our escape are interpreted correctly
1254
1255
1256
1257
1258
1259
1260
        return _load_chat_template(chat_template, is_literal=True)


_cached_load_chat_template = lru_cache(_load_chat_template)


def load_chat_template(
1261
    chat_template: Path | str | None,
1262
1263
    *,
    is_literal: bool = False,
1264
) -> str | None:
1265
    return _cached_load_chat_template(chat_template, is_literal=is_literal)
1266
1267


1268
1269
1270
def _get_interleaved_text_prompt(
    placeholder_storage: dict[str, list], texts: list[str]
) -> str:
1271
1272
1273
1274
1275
1276
1277
    for idx, elem in enumerate(texts):
        if elem in placeholder_storage:
            texts[idx] = placeholder_storage[elem].pop(0)

    return "\n".join(texts)


1278
# TODO: Let user specify how to insert multimodal tokens into prompt
1279
# (similar to chat template)
1280
1281
1282
1283
1284
def _get_full_multimodal_text_prompt(
    placeholder_storage: dict[str, list],
    texts: list[str],
    interleave_strings: bool,
) -> str:
1285
    """Combine multimodal prompts for a multimodal language model."""
1286

1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
    # flatten storage to make it looks like
    # {
    #   "<|image|>": 2,
    #   "<|audio|>": 1
    # }
    placeholder_counts = Counter(
        [v for elem in placeholder_storage.values() for v in elem]
    )

    if interleave_strings:
        text_prompt = _get_interleaved_text_prompt(placeholder_storage, texts)
    else:
        text_prompt = "\n".join(texts)

    # Pass interleaved text further in case the user used image placeholders
    # himself, but forgot to disable the 'interleave_strings' flag

1304
    # Look through the text prompt to check for missing placeholders
1305
    missing_placeholders: list[str] = []
1306
1307
1308
1309
1310
    for placeholder in placeholder_counts:
        # For any existing placeholder in the text prompt, we leave it as is
        placeholder_counts[placeholder] -= text_prompt.count(placeholder)

        if placeholder_counts[placeholder] < 0:
1311
1312
1313
1314
            logger.error(
                "Placeholder count is negative! "
                "Ensure that the 'interleave_strings' flag is disabled "
                "(current value: %s) "
1315
1316
                "when manually placing image placeholders.",
                interleave_strings,
1317
1318
            )
            logger.debug("Input prompt: %s", text_prompt)
1319
1320
            raise ValueError(
                f"Found more '{placeholder}' placeholders in input prompt than "
1321
1322
                "actual multimodal data items."
            )
1323

1324
        missing_placeholders.extend([placeholder] * placeholder_counts[placeholder])
1325

1326
1327
    # NOTE: Default behaviour: we always add missing placeholders
    # at the front of the prompt, if interleave_strings=False
1328
    return "\n".join(missing_placeholders + [text_prompt])
1329
1330


1331
1332
# No need to validate using Pydantic again
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
1333
_ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
1334
_AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam)
1335
_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
1336
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
1337
_PILImageParser = partial(cast, CustomChatCompletionContentPILImageParam)
Julien Denize's avatar
Julien Denize committed
1338
_ThinkParser = partial(cast, CustomThinkCompletionContentParam)
1339
1340
1341
1342
# Need to validate url objects
_ImageParser = TypeAdapter(ChatCompletionContentPartImageParam).validate_python
_AudioParser = TypeAdapter(ChatCompletionContentPartAudioParam).validate_python
_VideoParser = TypeAdapter(ChatCompletionContentPartVideoParam).validate_python
1343

1344
_ResponsesInputImageParser = TypeAdapter(ResponseInputImageParam).validate_python
1345
_ContentPart: TypeAlias = str | dict[str, str] | InputAudio | PILImage
1346

1347
# Define a mapping from part types to their corresponding parsing functions.
1348
MM_PARSER_MAP: dict[
1349
1350
1351
    str,
    Callable[[ChatCompletionContentPartParam], _ContentPart],
] = {
1352
1353
1354
    "text": lambda part: _TextParser(part).get("text", None),
    "thinking": lambda part: _ThinkParser(part).get("thinking", None),
    "input_text": lambda part: _TextParser(part).get("text", None),
1355
    "output_text": lambda part: _TextParser(part).get("text", None),
1356
1357
1358
    "input_image": lambda part: _ResponsesInputImageParser(part).get("image_url", None),
    "image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
    "image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
1359
    "audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None),
1360
    "image_pil": lambda part: _PILImageParser(part).get("image_pil", None),
1361
1362
    "audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
    "input_audio": lambda part: _InputAudioParser(part).get("input_audio", None),
1363
    "refusal": lambda part: _RefusalParser(part).get("refusal", None),
1364
    "video_url": lambda part: _VideoParser(part).get("video_url", {}).get("url", None),
1365
1366
1367
1368
}


def _parse_chat_message_content_mm_part(
1369
1370
    part: ChatCompletionContentPartParam,
) -> tuple[str, _ContentPart]:
1371
    """
1372
    Parses a given multi-modal content part based on its type.
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385

    Args:
        part: A dict containing the content part, with a potential 'type' field.

    Returns:
        A tuple (part_type, content) where:
        - part_type: Type of the part (e.g., 'text', 'image_url').
        - content: Parsed content (e.g., text, image URL).

    Raises:
        ValueError: If the 'type' field is missing and no direct URL is found.
    """
    assert isinstance(
1386
1387
        part, dict
    )  # This is needed to avoid mypy errors: part.get() from str
1388
    part_type = part.get("type", None)
1389
    uuid = part.get("uuid", None)
1390

1391
    if isinstance(part_type, str) and part_type in MM_PARSER_MAP and uuid is None:  # noqa: E501
1392
1393
1394
        content = MM_PARSER_MAP[part_type](part)

        # Special case for 'image_url.detail'
1395
1396
        # We only support 'auto', which is the default
        if part_type == "image_url" and part.get("detail", "auto") != "auto":
1397
            logger.warning(
1398
                "'image_url.detail' is currently not supported and will be ignored."
1399
            )
1400
1401
1402
1403

        return part_type, content

    # Handle missing 'type' but provided direct URL fields.
1404
    # 'type' is required field by pydantic
1405
1406
    if part_type is None or uuid is not None:
        if "image_url" in part:
1407
            image_params = cast(CustomChatCompletionContentSimpleImageParam, part)
1408
1409
1410
1411
1412
1413
1414
1415
            image_url = image_params.get("image_url", None)
            if isinstance(image_url, dict):
                # Can potentially happen if user provides a uuid
                # with url as a dict of {"url": url}
                image_url = image_url.get("url", None)
            return "image_url", image_url
        if "image_pil" in part:
            # "image_pil" could be None if UUID is provided.
1416
            image_params = cast(  # type: ignore
1417
1418
1419
1420
1421
1422
                CustomChatCompletionContentPILImageParam, part
            )
            image_pil = image_params.get("image_pil", None)
            return "image_pil", image_pil
        if "image_embeds" in part:
            # "image_embeds" could be None if UUID is provided.
1423
            image_params = cast(  # type: ignore
1424
1425
1426
1427
                ChatCompletionContentPartImageEmbedsParam, part
            )
            image_embeds = image_params.get("image_embeds", None)
            return "image_embeds", image_embeds
1428
1429
1430
1431
1432
1433
1434
        if "audio_embeds" in part:
            # "audio_embeds" could be None if UUID is provided.
            audio_params = cast(  # type: ignore[assignment]
                ChatCompletionContentPartAudioEmbedsParam, part
            )
            audio_embeds = audio_params.get("audio_embeds", None)
            return "audio_embeds", audio_embeds
1435
        if "audio_url" in part:
1436
1437
1438
            audio_params = cast(  # type: ignore[assignment]
                CustomChatCompletionContentSimpleAudioParam, part
            )
1439
1440
1441
1442
1443
1444
            audio_url = audio_params.get("audio_url", None)
            if isinstance(audio_url, dict):
                # Can potentially happen if user provides a uuid
                # with url as a dict of {"url": url}
                audio_url = audio_url.get("url", None)
            return "audio_url", audio_url
1445
        if part.get("input_audio") is not None:
1446
            input_audio_params = cast(dict[str, str], part)
1447
            return "input_audio", input_audio_params
1448
        if "video_url" in part:
1449
            video_params = cast(CustomChatCompletionContentSimpleVideoParam, part)
1450
1451
1452
1453
1454
1455
            video_url = video_params.get("video_url", None)
            if isinstance(video_url, dict):
                # Can potentially happen if user provides a uuid
                # with url as a dict of {"url": url}
                video_url = video_url.get("url", None)
            return "video_url", video_url
1456
1457
1458
1459
1460
1461
1462
1463
        # Raise an error if no 'type' or direct URL is found.
        raise ValueError("Missing 'type' field in multimodal part.")

    if not isinstance(part_type, str):
        raise ValueError("Invalid 'type' field in multimodal part.")
    return part_type, "unknown part_type content"


1464
PART_TYPES_TO_SKIP_NONE_CONTENT = (
1465
1466
1467
    "text",
    "refusal",
)
1468

1469

1470
1471
1472
def _parse_chat_message_content_parts(
    role: str,
    parts: Iterable[ChatCompletionContentPartParam],
1473
    mm_tracker: BaseMultiModalItemTracker,
1474
1475
    *,
    wrap_dicts: bool,
1476
    interleave_strings: bool,
1477
) -> list[ConversationMessage]:
1478
    content = list[_ContentPart]()
1479

1480
    mm_parser = mm_tracker.create_parser()
1481
1482

    for part in parts:
1483
        parse_res = _parse_chat_message_content_part(
1484
1485
1486
            part,
            mm_parser,
            wrap_dicts=wrap_dicts,
1487
            interleave_strings=interleave_strings,
1488
        )
1489
1490
        if parse_res:
            content.append(parse_res)
1491

1492
    if wrap_dicts:
1493
        # Parsing wraps images and texts as interleaved dictionaries
1494
        return [ConversationMessage(role=role, content=content)]  # type: ignore
1495
    texts = cast(list[str], content)
1496
1497
    mm_placeholder_storage = mm_parser.mm_placeholder_storage()
    if mm_placeholder_storage:
1498
1499
1500
        text_prompt = _get_full_multimodal_text_prompt(
            mm_placeholder_storage, texts, interleave_strings
        )
1501
1502
1503
    else:
        text_prompt = "\n".join(texts)

1504
1505
1506
1507
    return [ConversationMessage(role=role, content=text_prompt)]


def _parse_chat_message_content_part(
1508
1509
1510
1511
    part: ChatCompletionContentPartParam,
    mm_parser: BaseMultiModalContentParser,
    *,
    wrap_dicts: bool,
1512
    interleave_strings: bool,
1513
) -> _ContentPart | None:
1514
1515
1516
1517
1518
1519
1520
1521
    """Parses a single part of a conversation. If wrap_dicts is True,
    structured dictionary pieces for texts and images will be
    wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
    {"type": "image"}, respectively. Otherwise multimodal data will be
    handled by mm_parser, and texts will be returned as strings to be joined
    with multimodal placeholders.
    """
    if isinstance(part, str):  # Handle plain text parts
1522
        return part
1523
1524
    # Handle structured dictionary parts
    part_type, content = _parse_chat_message_content_mm_part(part)
1525
    # if part_type is text/refusal/image_url/audio_url/video_url/input_audio but
1526
    # content is None, log a warning and skip
1527
    if part_type in PART_TYPES_TO_SKIP_NONE_CONTENT and content is None:
1528
        logger.warning(
1529
            "Skipping multimodal part '%s' (type: '%s') "
1530
1531
1532
1533
            "with empty / unparsable content.",
            part,
            part_type,
        )
1534
1535
        return None

1536
    if part_type in ("text", "input_text", "output_text", "refusal", "thinking"):
1537
1538
        str_content = cast(str, content)
        if wrap_dicts:
1539
            return {"type": "text", "text": str_content}
1540
1541
        else:
            return str_content
1542

1543
1544
1545
1546
1547
1548
    # For media items, if a user has provided one, use it. Otherwise, insert
    # a placeholder empty uuid.
    uuid = part.get("uuid", None)
    if uuid is not None:
        uuid = str(uuid)

1549
    modality = None
1550
    if part_type == "image_pil":
1551
        image_content = cast(Image.Image, content) if content is not None else None
1552
        mm_parser.parse_image_pil(image_content, uuid)
1553
        modality = "image"
1554
    elif part_type in ("image_url", "input_image"):
1555
        str_content = cast(str, content)
1556
        mm_parser.parse_image(str_content, uuid)
1557
1558
        modality = "image"
    elif part_type == "image_embeds":
1559
        content = cast(str | dict[str, str], content) if content is not None else None
1560
        mm_parser.parse_image_embeds(content, uuid)
1561
        modality = "image"
1562
1563
1564
1565
    elif part_type == "audio_embeds":
        content = cast(str | dict[str, str], content) if content is not None else None
        mm_parser.parse_audio_embeds(content, uuid)
        modality = "audio"
1566
    elif part_type == "audio_url":
1567
        str_content = cast(str, content)
1568
        mm_parser.parse_audio(str_content, uuid)
1569
1570
        modality = "audio"
    elif part_type == "input_audio":
1571
        dict_content = cast(InputAudio, content)
1572
        mm_parser.parse_input_audio(dict_content, uuid)
1573
1574
        modality = "audio"
    elif part_type == "video_url":
1575
        str_content = cast(str, content)
1576
        mm_parser.parse_video(str_content, uuid)
1577
1578
1579
        modality = "video"
    else:
        raise NotImplementedError(f"Unknown part type: {part_type}")
1580

1581
1582
1583
    return (
        {"type": modality}
        if wrap_dicts
1584
        else (MODALITY_PLACEHOLDERS_MAP[modality] if interleave_strings else None)
1585
    )
1586
1587


1588
1589
1590
1591
1592
# No need to validate using Pydantic again
_AssistantParser = partial(cast, ChatCompletionAssistantMessageParam)
_ToolParser = partial(cast, ChatCompletionToolMessageParam)


1593
def _parse_chat_message_content(
1594
1595
    message: ChatCompletionMessageParam,
    mm_tracker: BaseMultiModalItemTracker,
1596
    content_format: _ChatTemplateContentFormat,
1597
    interleave_strings: bool,
1598
) -> list[ConversationMessage]:
1599
1600
    role = message["role"]
    content = message.get("content")
1601
    reasoning = message.get("reasoning") or message.get("reasoning_content")
1602

1603
    if content is None:
1604
1605
        content = []
    elif isinstance(content, str):
1606
        content = [ChatCompletionContentPartTextParam(type="text", text=content)]
1607
    result = _parse_chat_message_content_parts(
1608
1609
        role,
        content,  # type: ignore
1610
        mm_tracker,
1611
        wrap_dicts=(content_format == "openai"),
1612
        interleave_strings=interleave_strings,
1613
    )
1614

1615
    for result_msg in result:
1616
        if role == "assistant":
1617
1618
            parsed_msg = _AssistantParser(message)

1619
1620
1621
            # The 'tool_calls' is not None check ensures compatibility.
            # It's needed only if downstream code doesn't strictly
            # follow the OpenAI spec.
1622
            if "tool_calls" in parsed_msg and parsed_msg["tool_calls"] is not None:
1623
                result_msg["tool_calls"] = list(parsed_msg["tool_calls"])
1624
1625
1626
1627
1628
1629
            # Include reasoning if present for interleaved thinking.
            if reasoning is not None:
                result_msg["reasoning"] = cast(str, reasoning)
                result_msg["reasoning_content"] = cast(
                    str, reasoning
                )  # keep compatibility
1630
1631
1632
1633
1634
1635
1636
1637
        elif role == "tool":
            parsed_msg = _ToolParser(message)
            if "tool_call_id" in parsed_msg:
                result_msg["tool_call_id"] = parsed_msg["tool_call_id"]

        if "name" in message and isinstance(message["name"], str):
            result_msg["name"] = message["name"]

1638
1639
        if role == "developer":
            result_msg["tools"] = message.get("tools", None)
1640
1641
    return result

1642

1643
def _postprocess_messages(messages: list[ConversationMessage]) -> None:
1644
1645
1646
1647
1648
1649
    # per the Transformers docs & maintainers, tool call arguments in
    # assistant-role messages with tool_calls need to be dicts not JSON str -
    # this is how tool-use chat templates will expect them moving forwards
    # so, for messages that have tool_calls, parse the string (which we get
    # from openAI format) to dict
    for message in messages:
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
        if message["role"] == "assistant" and "tool_calls" in message:
            tool_calls = message.get("tool_calls")
            if not isinstance(tool_calls, list):
                continue

            if len(tool_calls) == 0:
                # Drop empty tool_calls to keep templates on the normal assistant path.
                message.pop("tool_calls", None)
                continue

            for item in tool_calls:
1661
1662
                # if arguments is None or empty string, set to {}
                if content := item["function"].get("arguments"):
1663
1664
                    if not isinstance(content, (dict, list)):
                        item["function"]["arguments"] = json.loads(content)
1665
1666
                else:
                    item["function"]["arguments"] = {}
1667
1668


1669
def parse_chat_messages(
1670
    messages: list[ChatCompletionMessageParam],
1671
    model_config: ModelConfig,
1672
    content_format: _ChatTemplateContentFormat,
1673
1674
) -> tuple[
    list[ConversationMessage],
1675
1676
    MultiModalDataDict | None,
    MultiModalUUIDDict | None,
1677
]:
1678
    conversation: list[ConversationMessage] = []
1679
    mm_tracker = MultiModalItemTracker(model_config)
1680
1681

    for msg in messages:
1682
1683
1684
        sub_messages = _parse_chat_message_content(
            msg,
            mm_tracker,
1685
            content_format,
1686
1687
1688
1689
            interleave_strings=(
                content_format == "string"
                and model_config.multimodal_config is not None
                and model_config.multimodal_config.interleave_mm_strings
1690
            ),
1691
        )
1692

1693
        conversation.extend(sub_messages)
1694

1695
1696
    _postprocess_messages(conversation)

1697
    return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1698
1699


1700
def parse_chat_messages_futures(
1701
    messages: list[ChatCompletionMessageParam],
1702
    model_config: ModelConfig,
1703
    content_format: _ChatTemplateContentFormat,
1704
1705
) -> tuple[
    list[ConversationMessage],
1706
1707
    Awaitable[MultiModalDataDict | None],
    MultiModalUUIDDict | None,
1708
]:
1709
    conversation: list[ConversationMessage] = []
1710
    mm_tracker = AsyncMultiModalItemTracker(model_config)
1711
1712

    for msg in messages:
1713
1714
1715
        sub_messages = _parse_chat_message_content(
            msg,
            mm_tracker,
1716
            content_format,
1717
1718
1719
1720
            interleave_strings=(
                content_format == "string"
                and model_config.multimodal_config is not None
                and model_config.multimodal_config.interleave_mm_strings
1721
            ),
1722
        )
1723
1724
1725

        conversation.extend(sub_messages)

1726
1727
    _postprocess_messages(conversation)

1728
    return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
1729
1730


1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
# adapted from https://github.com/huggingface/transformers/blob/v4.56.2/src/transformers/utils/chat_template_utils.py#L398-L412
# only preserve the parse function used to resolve chat template kwargs
class AssistantTracker(jinja2.ext.Extension):
    tags = {"generation"}

    def parse(self, parser: jinja2.parser.Parser) -> jinja2.nodes.CallBlock:
        lineno = next(parser.stream).lineno
        body = parser.parse_statements(["name:endgeneration"], drop_needle=True)
        call = self.call_method("_generation_support")
        call_block = jinja2.nodes.CallBlock(call, [], [], body)
        return call_block.set_lineno(lineno)


1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
def _resolve_chat_template_kwargs(
    chat_template: str,
):
    env = jinja2.sandbox.ImmutableSandboxedEnvironment(
        trim_blocks=True,
        lstrip_blocks=True,
        extensions=[AssistantTracker, jinja2.ext.loopcontrols],
    )
    parsed_content = env.parse(chat_template)
    template_vars = jinja2.meta.find_undeclared_variables(parsed_content)
    return template_vars


_cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs)


1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
@lru_cache
def _get_hf_base_chat_template_params() -> frozenset[str]:
    # Get standard parameters from HuggingFace's base tokenizer class.
    # This dynamically extracts parameters from PreTrainedTokenizer's
    # apply_chat_template method, ensuring compatibility with tokenizers
    # that use **kwargs to receive standard parameters.

    # Read signature from HF's base class - the single source of truth
    base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template)
    # Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders
    return frozenset(
        p.name
        for p in base_sig.parameters.values()
        if p.kind
        not in (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL)
    )


1778
def resolve_chat_template_kwargs(
1779
    tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
1780
1781
    chat_template: str,
    chat_template_kwargs: dict[str, Any],
1782
    raise_on_unexpected: bool = True,
1783
) -> dict[str, Any]:
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
    # We exclude chat_template from kwargs here, because
    # chat template has been already resolved at this stage
    unexpected_vars = {"chat_template", "tokenize"}
    if raise_on_unexpected and (
        unexpected_in_kwargs := unexpected_vars & chat_template_kwargs.keys()
    ):
        raise ValueError(
            "Found unexpected chat template kwargs from request: "
            f"{unexpected_in_kwargs}"
        )

1795
    fn_kw = {
1796
1797
        k
        for k in chat_template_kwargs
1798
1799
        if supports_kw(tokenizer.apply_chat_template, k, allow_var_kwargs=False)
    }
1800
    template_vars = _cached_resolve_chat_template_kwargs(chat_template)
1801
1802
1803
1804
1805

    # Allow standard HF parameters even if tokenizer uses **kwargs to receive them
    hf_base_params = _get_hf_base_chat_template_params()

    accept_vars = (fn_kw | template_vars | hf_base_params) - unexpected_vars
1806
    return {k: v for k, v in chat_template_kwargs.items() if k in accept_vars}
1807
1808


1809
def apply_hf_chat_template(
1810
    tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast,
1811
    conversation: list[ConversationMessage],
1812
1813
    chat_template: str | None,
    tools: list[dict[str, Any]] | None,
1814
    *,
1815
    model_config: ModelConfig,
1816
    **kwargs: Any,
1817
) -> str:
1818
    hf_chat_template = resolve_hf_chat_template(
1819
1820
1821
        tokenizer,
        chat_template=chat_template,
        tools=tools,
1822
        model_config=model_config,
1823
    )
1824

1825
    if hf_chat_template is None:
1826
        raise ChatTemplateResolutionError(
1827
1828
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
1829
1830
            "does not define one."
        )
1831

1832
1833
1834
1835
1836
1837
    resolved_kwargs = resolve_chat_template_kwargs(
        tokenizer=tokenizer,
        chat_template=hf_chat_template,
        chat_template_kwargs=kwargs,
    )

1838
1839
1840
1841
1842
    try:
        return tokenizer.apply_chat_template(
            conversation=conversation,  # type: ignore[arg-type]
            tools=tools,  # type: ignore[arg-type]
            chat_template=hf_chat_template,
1843
            tokenize=False,
1844
            **resolved_kwargs,
1845
        )
1846

1847
1848
1849
1850
1851
1852
    # External library exceptions can sometimes occur despite the framework's
    # internal exception management capabilities.
    except Exception as e:
        # Log and report any library-related exceptions for further
        # investigation.
        logger.exception(
1853
1854
            "An error occurred in `transformers` while applying chat template"
        )
1855
        raise ValueError(str(e)) from e
1856

1857

1858
def apply_mistral_chat_template(
1859
    tokenizer: "MistralTokenizer",
1860
    messages: list[ChatCompletionMessageParam],
1861
1862
    chat_template: str | None,
    tools: list[dict[str, Any]] | None,
1863
    **kwargs: Any,
1864
) -> list[int]:
1865
1866
    from mistral_common.exceptions import MistralCommonException

1867
1868
1869
1870
1871
1872
    # The return value of resolve_mistral_chat_template is always None,
    # and we won't use it.
    resolve_mistral_chat_template(
        chat_template=chat_template,
        **kwargs,
    )
1873

1874
1875
1876
1877
1878
1879
1880
1881
1882
    try:
        return tokenizer.apply_chat_template(
            messages=messages,
            tools=tools,
            **kwargs,
        )
    # mistral-common uses assert statements to stop processing of input
    # if input does not comply with the expected format.
    # We convert those assertion errors to ValueErrors so they can be
1883
    # properly caught in the preprocessing_input step
1884
    except (AssertionError, MistralCommonException) as e:
1885
        raise ValueError(str(e)) from e
1886
1887
1888
1889
1890
1891
1892

    # External library exceptions can sometimes occur despite the framework's
    # internal exception management capabilities.
    except Exception as e:
        # Log and report any library-related exceptions for further
        # investigation.
        logger.exception(
1893
1894
            "An error occurred in `mistral_common` while applying chat template"
        )
1895
        raise ValueError(str(e)) from e
1896

1897

1898
1899
1900
def get_history_tool_calls_cnt(conversation: list[ConversationMessage]):
    idx = 0
    for msg in conversation:
1901
1902
1903
        if msg["role"] == "assistant":
            tool_calls = msg.get("tool_calls")
            idx += len(list(tool_calls)) if tool_calls is not None else 0  # noqa
1904
1905
1906
    return idx


1907
1908
1909
def make_tool_call_id(id_type: str = "random", func_name=None, idx=None):
    if id_type == "kimi_k2":
        return f"functions.{func_name}:{idx}"
1910
1911
1912
    else:
        # by default return random
        return f"chatcmpl-tool-{random_uuid()}"