chat_utils.py 25.4 KB
Newer Older
1
import asyncio
2
import codecs
3
import json
4
from abc import ABC, abstractmethod
5
from collections import defaultdict
6
from functools import lru_cache, partial
7
from pathlib import Path
8
9
from typing import (Any, Awaitable, Callable, Dict, Generic, Iterable, List,
                    Literal, Mapping, Optional, Tuple, TypeVar, Union, cast)
10

11
12
# yapf conflicts with isort for this block
# yapf: disable
13
14
from openai.types.chat import (ChatCompletionAssistantMessageParam,
                               ChatCompletionContentPartImageParam)
15
16
from openai.types.chat import (
    ChatCompletionContentPartParam as OpenAIChatCompletionContentPartParam)
17
18
from openai.types.chat import (ChatCompletionContentPartRefusalParam,
                               ChatCompletionContentPartTextParam)
19
20
from openai.types.chat import (
    ChatCompletionMessageParam as OpenAIChatCompletionMessageParam)
21
22
from openai.types.chat import (ChatCompletionMessageToolCallParam,
                               ChatCompletionToolMessageParam)
23
24
# yapf: enable
# pydantic needs the TypedDict from typing_extensions
25
from pydantic import ConfigDict
26
from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
27
from typing_extensions import Required, TypeAlias, TypedDict
28

29
from vllm.config import ModelConfig
30
31
from vllm.logger import init_logger
from vllm.multimodal import MultiModalDataDict
32
from vllm.multimodal.utils import (async_get_and_parse_audio,
33
34
                                   async_get_and_parse_image,
                                   get_and_parse_audio, get_and_parse_image)
35
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
36
from vllm.utils import print_warning_once
37
38
39
40

logger = init_logger(__name__)


41
42
43
44
45
46
47
48
49
50
51
52
53
54
class AudioURL(TypedDict, total=False):
    url: Required[str]
    """
    Either a URL of the audio or a data URL with base64 encoded audio data.
    """


class ChatCompletionContentPartAudioParam(TypedDict, total=False):
    audio_url: Required[AudioURL]

    type: Required[Literal["audio_url"]]
    """The type of the content part."""


55
56
57
58
59
60
61
class CustomChatCompletionContentPartParam(TypedDict, total=False):
    __pydantic_config__ = ConfigDict(extra="allow")  # type: ignore

    type: Required[str]
    """The type of the content part."""


62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
class CustomChatCompletionContentSimpleImageParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain image_url.
    This is supported by OpenAI API, although it is not documented.
    
    Example:
    {
        "image_url": "https://example.com/image.jpg"
    }
    """
    image_url: Required[str]


class CustomChatCompletionContentSimpleAudioParam(TypedDict, total=False):
    """A simpler version of the param that only accepts a plain audio_url.
    
    Example:
    {
        "audio_url": "https://example.com/audio.mp3"
    }
    """
    audio_url: Required[str]


85
86
ChatCompletionContentPartParam: TypeAlias = Union[
    OpenAIChatCompletionContentPartParam, ChatCompletionContentPartAudioParam,
87
    ChatCompletionContentPartRefusalParam,
88
89
90
    CustomChatCompletionContentPartParam,
    CustomChatCompletionContentSimpleImageParam,
    CustomChatCompletionContentSimpleAudioParam, str]
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107


class CustomChatCompletionMessageParam(TypedDict, total=False):
    """Enables custom roles in the Chat Completion API."""
    role: Required[str]
    """The role of the message's author."""

    content: Union[str, List[ChatCompletionContentPartParam]]
    """The contents of the message."""

    name: str
    """An optional name for the participant.

    Provides the model information to differentiate between participants of the
    same role.
    """

108
109
110
111
112
113
    tool_call_id: Optional[str]
    """Tool call that this message is responding to."""

    tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
    """The tool calls generated by the model, such as function calls."""

114
115
116
117
118

ChatCompletionMessageParam = Union[OpenAIChatCompletionMessageParam,
                                   CustomChatCompletionMessageParam]


119
# TODO: Make fields ReadOnly once mypy supports it
120
121
122
123
class ConversationMessage(TypedDict, total=False):
    role: Required[str]
    """The role of the message's author."""

124
    content: Union[Optional[str], List[Dict[str, str]]]
125
126
127
128
129
130
131
132
133
134
    """The contents of the message"""

    tool_call_id: Optional[str]
    """Tool call that this message is responding to."""

    name: Optional[str]
    """The name of the function to call"""

    tool_calls: Optional[Iterable[ChatCompletionMessageToolCallParam]]
    """The tool calls generated by the model, such as function calls."""
135
136


137
ModalityStr = Literal["image", "audio", "video"]
138
139
140
141
_T = TypeVar("_T")


class BaseMultiModalItemTracker(ABC, Generic[_T]):
142
143
144
145
146
147
148
    """
    Tracks multi-modal items in a given request and ensures that the number
    of multi-modal items in a given request does not exceed the configured
    maximum per prompt.
    """

    def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer):
149
150
        super().__init__()

151
152
153
154
155
        self._model_config = model_config
        self._tokenizer = tokenizer
        self._allowed_items = (model_config.multimodal_config.limit_per_prompt
                               if model_config.multimodal_config else {})
        self._consumed_items = {k: 0 for k in self._allowed_items}
156
157

        self._items: List[_T] = []
158

159
160
161
162
    @property
    def model_config(self) -> ModelConfig:
        return self._model_config

163
164
    @staticmethod
    @lru_cache(maxsize=None)
165
    def _cached_token_str(tokenizer: AnyTokenizer, token_index: int) -> str:
166
167
        return tokenizer.decode(token_index)

168
169
    def _placeholder_str(self, modality: ModalityStr,
                         current_count: int) -> Optional[str]:
170
171
        # TODO: Let user specify how to insert image tokens into prompt
        # (similar to chat template)
172
173
174
        hf_config = self._model_config.hf_config
        model_type = hf_config.model_type

175
176
177
178
179
180
        if modality == "image":
            if model_type == "phi3_v":
                # Workaround since this token is not defined in the tokenizer
                return f"<|image_{current_count}|>"
            if model_type == "minicpmv":
                return "(<image>./</image>)"
Patrick von Platen's avatar
Patrick von Platen committed
181
182
            if model_type in ("blip-2", "chatglm", "fuyu", "paligemma",
                              "pixtral"):
183
184
                # These models do not use image tokens in the prompt
                return None
185
186
            if model_type == "qwen":
                return f"Picture {current_count}: <img></img>"
187
            if model_type.startswith("llava"):
188
189
                return self._cached_token_str(self._tokenizer,
                                              hf_config.image_token_index)
190
191
            if model_type in ("chameleon", "internvl_chat", "NVLM_D",
                              "h2ovl_chat"):
192
                return "<image>"
193
194
            if model_type == "mllama":
                return "<|image|>"
195
196
            if model_type == "qwen2_vl":
                return "<|vision_start|><|image_pad|><|vision_end|>"
197
198
            if model_type == "molmo":
                return ""
199

200
            raise TypeError(f"Unknown {modality} model type: {model_type}")
201
202
203
        elif modality == "audio":
            if model_type == "ultravox":
                return "<|reserved_special_token_0|>"
204
205
206
207
            if model_type == "qwen2_audio":
                return (f"Audio {current_count}: "
                        f"<|audio_bos|><|AUDIO|><|audio_eos|>")
            raise TypeError(f"Unknown model type: {model_type}")
208
209
210
        elif modality == "video":
            if model_type == "qwen2_vl":
                return "<|vision_start|><|video_pad|><|vision_end|>"
211
            raise TypeError(f"Unknown {modality} model type: {model_type}")
212
213
214
215
        else:
            raise TypeError(f"Unknown modality: {modality}")

    @staticmethod
216
    def _combine(items: List[MultiModalDataDict]) -> MultiModalDataDict:
217
218
219
        mm_lists: Mapping[str, List[object]] = defaultdict(list)

        # Merge all the multi-modal items
220
        for single_mm_data in items:
221
222
223
224
225
226
227
228
229
230
231
232
            for mm_key, mm_item in single_mm_data.items():
                if isinstance(mm_item, list):
                    mm_lists[mm_key].extend(mm_item)
                else:
                    mm_lists[mm_key].append(mm_item)

        # Unpack any single item lists for models that don't expect multiple.
        return {
            mm_key: mm_list[0] if len(mm_list) == 1 else mm_list
            for mm_key, mm_list in mm_lists.items()
        }

233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
    def add(self, modality: ModalityStr, item: _T) -> Optional[str]:
        """
        Add a multi-modal item to the current prompt and returns the
        placeholder string to use, if any.
        """
        allowed_count = self._allowed_items.get(modality, 1)
        current_count = self._consumed_items.get(modality, 0) + 1
        if current_count > allowed_count:
            raise ValueError(
                f"At most {allowed_count} {modality}(s) may be provided in "
                "one request.")

        self._consumed_items[modality] = current_count
        self._items.append(item)

        return self._placeholder_str(modality, current_count)

    @abstractmethod
    def create_parser(self) -> "BaseMultiModalContentParser":
        raise NotImplementedError


class MultiModalItemTracker(BaseMultiModalItemTracker[MultiModalDataDict]):

    def all_mm_data(self) -> Optional[MultiModalDataDict]:
        return self._combine(self._items) if self._items else None

    def create_parser(self) -> "BaseMultiModalContentParser":
        return MultiModalContentParser(self)


class AsyncMultiModalItemTracker(
        BaseMultiModalItemTracker[Awaitable[MultiModalDataDict]]):

    async def all_mm_data(self) -> Optional[MultiModalDataDict]:
        if self._items:
            items = await asyncio.gather(*self._items)
            return self._combine(items)

        return None

    def create_parser(self) -> "BaseMultiModalContentParser":
        return AsyncMultiModalContentParser(self)


class BaseMultiModalContentParser(ABC):

    def __init__(self) -> None:
        super().__init__()

        # multimodal placeholder_string : count
        self._placeholder_counts: Dict[str, int] = defaultdict(lambda: 0)

    def _add_placeholder(self, placeholder: Optional[str]):
        if placeholder:
            self._placeholder_counts[placeholder] += 1

    def mm_placeholder_counts(self) -> Dict[str, int]:
        return dict(self._placeholder_counts)

    @abstractmethod
    def parse_image(self, image_url: str) -> None:
        raise NotImplementedError

    @abstractmethod
    def parse_audio(self, audio_url: str) -> None:
        raise NotImplementedError


class MultiModalContentParser(BaseMultiModalContentParser):

    def __init__(self, tracker: MultiModalItemTracker) -> None:
        super().__init__()

        self._tracker = tracker

    def parse_image(self, image_url: str) -> None:
310
311
312
        image = get_and_parse_image(image_url,
                                    allowed_local_media_path=self._tracker.
                                    _model_config.allowed_local_media_path)
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331

        placeholder = self._tracker.add("image", image)
        self._add_placeholder(placeholder)

    def parse_audio(self, audio_url: str) -> None:
        audio = get_and_parse_audio(audio_url)

        placeholder = self._tracker.add("audio", audio)
        self._add_placeholder(placeholder)


class AsyncMultiModalContentParser(BaseMultiModalContentParser):

    def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:
        super().__init__()

        self._tracker = tracker

    def parse_image(self, image_url: str) -> None:
332
333
334
335
        image_coro = async_get_and_parse_image(
            image_url,
            allowed_local_media_path=self._tracker._model_config.
            allowed_local_media_path)
336
337
338
339
340
341
342
343
344

        placeholder = self._tracker.add("image", image_coro)
        self._add_placeholder(placeholder)

    def parse_audio(self, audio_url: str) -> None:
        audio_coro = async_get_and_parse_audio(audio_url)

        placeholder = self._tracker.add("audio", audio_coro)
        self._add_placeholder(placeholder)
345
346


347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
def validate_chat_template(chat_template: Optional[Union[Path, str]]):
    """Raises if the provided chat template appears invalid."""
    if chat_template is None:
        return

    elif isinstance(chat_template, Path) and not chat_template.exists():
        raise FileNotFoundError(
            "the supplied chat template path doesn't exist")

    elif isinstance(chat_template, str):
        JINJA_CHARS = "{}\n"
        if not any(c in chat_template
                   for c in JINJA_CHARS) and not Path(chat_template).exists():
            raise ValueError(
                f"The supplied chat template string ({chat_template}) "
                f"appears path-like, but doesn't exist!")

    else:
        raise TypeError(
            f"{type(chat_template)} is not a valid chat template type")


369
370
def load_chat_template(
        chat_template: Optional[Union[Path, str]]) -> Optional[str]:
371
372
373
374
375
376
    if chat_template is None:
        return None
    try:
        with open(chat_template, "r") as f:
            resolved_chat_template = f.read()
    except OSError as e:
377
378
379
        if isinstance(chat_template, Path):
            raise

380
381
382
383
384
385
        JINJA_CHARS = "{}\n"
        if not any(c in chat_template for c in JINJA_CHARS):
            msg = (f"The supplied chat template ({chat_template}) "
                   f"looks like a file path, but it failed to be "
                   f"opened. Reason: {e}")
            raise ValueError(msg) from e
386

387
388
389
        # If opening a file fails, set chat template to be args to
        # ensure we decode so our escape are interpreted correctly
        resolved_chat_template = codecs.decode(chat_template, "unicode_escape")
390

391
392
    logger.info("Using supplied chat template:\n%s", resolved_chat_template)
    return resolved_chat_template
393
394


395
# TODO: Let user specify how to insert multimodal tokens into prompt
396
# (similar to chat template)
397
def _get_full_multimodal_text_prompt(placeholder_counts: Dict[str, int],
398
                                     text_prompt: str) -> str:
399
    """Combine multimodal prompts for a multimodal language model."""
400

401
    # Look through the text prompt to check for missing placeholders
402
    missing_placeholders: List[str] = []
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
    for placeholder in placeholder_counts:

        # For any existing placeholder in the text prompt, we leave it as is
        placeholder_counts[placeholder] -= text_prompt.count(placeholder)

        if placeholder_counts[placeholder] < 0:
            raise ValueError(
                f"Found more '{placeholder}' placeholders in input prompt than "
                "actual multimodal data items.")

        missing_placeholders.extend([placeholder] *
                                    placeholder_counts[placeholder])

    # NOTE: For now we always add missing placeholders at the front of
    # the prompt. This may change to be customizable in the future.
    return "\n".join(missing_placeholders + [text_prompt])
419
420


421
422
423
424
425
# No need to validate using Pydantic again
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
_ImageParser = partial(cast, ChatCompletionContentPartImageParam)
_AudioParser = partial(cast, ChatCompletionContentPartAudioParam)
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
426
MODEL_KEEP_MULTI_MODAL_CONTENT = {'mllama'}
427

428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
# Define a mapping from part types to their corresponding parsing functions.
MM_PARSER_MAP: Dict[str, Callable[[ChatCompletionContentPartParam], str]] = {
    "text":
    lambda part: _TextParser(part).get("text", ""),
    "image_url":
    lambda part: _ImageParser(part).get("image_url", {}).get("url", ""),
    "audio_url":
    lambda part: _AudioParser(part).get("audio_url", {}).get("url", ""),
    "refusal":
    lambda part: _RefusalParser(part).get("refusal", ""),
}


def _parse_chat_message_content_mm_part(
        part: ChatCompletionContentPartParam) -> Tuple[str, str]:
    """
444
    Parses a given multi-modal content part based on its type.
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464

    Args:
        part: A dict containing the content part, with a potential 'type' field.

    Returns:
        A tuple (part_type, content) where:
        - part_type: Type of the part (e.g., 'text', 'image_url').
        - content: Parsed content (e.g., text, image URL).

    Raises:
        ValueError: If the 'type' field is missing and no direct URL is found.
    """
    assert isinstance(
        part, dict)  # This is needed to avoid mypy errors: part.get() from str
    part_type = part.get("type", None)

    if isinstance(part_type, str) and part_type in MM_PARSER_MAP:
        content = MM_PARSER_MAP[part_type](part)

        # Special case for 'image_url.detail'
465
466
        # We only support 'auto', which is the default
        if part_type == "image_url" and part.get("detail", "auto") != "auto":
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
            logger.warning("'image_url.detail' is currently not supported "
                           "and will be ignored.")

        return part_type, content

    # Handle missing 'type' but provided direct URL fields.
    if part_type is None:
        if part.get("image_url") is not None:
            image_params = cast(CustomChatCompletionContentSimpleImageParam,
                                part)
            return "image_url", image_params.get("image_url", "")
        if part.get("audio_url") is not None:
            audio_params = cast(CustomChatCompletionContentSimpleAudioParam,
                                part)
            return "audio_url", audio_params.get("audio_url", "")

        # Raise an error if no 'type' or direct URL is found.
        raise ValueError("Missing 'type' field in multimodal part.")

    if not isinstance(part_type, str):
        raise ValueError("Invalid 'type' field in multimodal part.")
    return part_type, "unknown part_type content"


VALID_MESSAGE_CONTENT_MM_PART_TYPES = ("text", "refusal", "image_url",
                                       "audio_url")

494

495
496
497
def _parse_chat_message_content_parts(
    role: str,
    parts: Iterable[ChatCompletionContentPartParam],
498
    mm_tracker: BaseMultiModalItemTracker,
499
    chat_template_text_format: str,
500
) -> List[ConversationMessage]:
501
    content: List[Union[str, Dict[str, str]]] = []
502

503
    mm_parser = mm_tracker.create_parser()
504
505
506
507
508
509
510
    model_config = mm_tracker.model_config

    wrap_dicts = (chat_template_text_format == "openai"
                  or (model_config.task == "embedding"
                      and model_config.is_multimodal_model)
                  or (model_config.hf_config.model_type
                      in MODEL_KEEP_MULTI_MODAL_CONTENT))
511
512

    for part in parts:
513
        parse_res = _parse_chat_message_content_part(
514
515
516
517
            part,
            mm_parser,
            wrap_dicts=wrap_dicts,
        )
518
519
        if parse_res:
            content.append(parse_res)
520

521
    if wrap_dicts:
522
        # Parsing wraps images and texts as interleaved dictionaries
523
        return [ConversationMessage(role=role,
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
                                    content=content)]  # type: ignore
    texts = cast(List[str], content)
    text_prompt = "\n".join(texts)
    mm_placeholder_counts = mm_parser.mm_placeholder_counts()
    if mm_placeholder_counts:
        text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_counts,
                                                       text_prompt)
    return [ConversationMessage(role=role, content=text_prompt)]


def _parse_chat_message_content_part(
        part: ChatCompletionContentPartParam,
        mm_parser: BaseMultiModalContentParser,
        wrap_dicts: bool) -> Optional[Union[str, Dict[str, str]]]:
    """Parses a single part of a conversation. If wrap_dicts is True,
    structured dictionary pieces for texts and images will be
    wrapped in dictionaries, i.e., {"type": "text", "text", ...} and
    {"type": "image"}, respectively. Otherwise multimodal data will be
    handled by mm_parser, and texts will be returned as strings to be joined
    with multimodal placeholders.
    """
    if isinstance(part, str):  # Handle plain text parts
        text = _TextParser(part)
        return text

    # Handle structured dictionary parts
    part_type, content = _parse_chat_message_content_mm_part(part)

    # if part_type is text/refusal/image_url/audio_url but
    # content is empty, log a warning and skip
    if part_type in VALID_MESSAGE_CONTENT_MM_PART_TYPES and not content:
        logger.warning(
            "Skipping multimodal part (type: '%s')"
            "with empty / unparsable content.", part_type)
        return None

    if part_type in ("text", "refusal"):
        return {'type': 'text', 'text': content} if wrap_dicts else content

    if part_type == "image_url":
        mm_parser.parse_image(content)
        return {'type': 'image'} if wrap_dicts else None

    if part_type == "audio_url":
        mm_parser.parse_audio(content)
        return {'type': 'audio'} if wrap_dicts else None

    raise NotImplementedError(f"Unknown part type: {part_type}")
572
573


574
575
576
577
578
# No need to validate using Pydantic again
_AssistantParser = partial(cast, ChatCompletionAssistantMessageParam)
_ToolParser = partial(cast, ChatCompletionToolMessageParam)


579
def _parse_chat_message_content(
580
581
    message: ChatCompletionMessageParam,
    mm_tracker: BaseMultiModalItemTracker,
582
    chat_template_text_format: str,
583
) -> List[ConversationMessage]:
584
585
586
587
    role = message["role"]
    content = message.get("content")

    if content is None:
588
589
590
591
592
        content = []
    elif isinstance(content, str):
        content = [
            ChatCompletionContentPartTextParam(type="text", text=content)
        ]
593

594
    result = _parse_chat_message_content_parts(
595
596
        role,
        content,  # type: ignore
597
        mm_tracker,
598
        chat_template_text_format,
599
    )
600

601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
    for result_msg in result:
        if role == 'assistant':
            parsed_msg = _AssistantParser(message)

            if "tool_calls" in parsed_msg:
                result_msg["tool_calls"] = list(parsed_msg["tool_calls"])
        elif role == "tool":
            parsed_msg = _ToolParser(message)
            if "tool_call_id" in parsed_msg:
                result_msg["tool_call_id"] = parsed_msg["tool_call_id"]

        if "name" in message and isinstance(message["name"], str):
            result_msg["name"] = message["name"]

    return result

617

618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
def _postprocess_messages(messages: List[ConversationMessage]) -> None:
    # per the Transformers docs & maintainers, tool call arguments in
    # assistant-role messages with tool_calls need to be dicts not JSON str -
    # this is how tool-use chat templates will expect them moving forwards
    # so, for messages that have tool_calls, parse the string (which we get
    # from openAI format) to dict
    for message in messages:
        if (message["role"] == "assistant" and "tool_calls" in message
                and isinstance(message["tool_calls"], list)):

            for item in message["tool_calls"]:
                item["function"]["arguments"] = json.loads(
                    item["function"]["arguments"])


633
634
635
def parse_chat_messages(
    messages: List[ChatCompletionMessageParam],
    model_config: ModelConfig,
636
    tokenizer: AnyTokenizer,
637
) -> Tuple[List[ConversationMessage], Optional[MultiModalDataDict]]:
638
    conversation: List[ConversationMessage] = []
639
    mm_tracker = MultiModalItemTracker(model_config, tokenizer)
640
641

    for msg in messages:
642
643
644
645
646
        sub_messages = _parse_chat_message_content(
            msg,
            mm_tracker,
            model_config.chat_template_text_format,
        )
647

648
        conversation.extend(sub_messages)
649

650
651
    _postprocess_messages(conversation)

652
    return conversation, mm_tracker.all_mm_data()
653
654


655
656
657
658
659
660
661
662
663
def parse_chat_messages_futures(
    messages: List[ChatCompletionMessageParam],
    model_config: ModelConfig,
    tokenizer: AnyTokenizer,
) -> Tuple[List[ConversationMessage], Awaitable[Optional[MultiModalDataDict]]]:
    conversation: List[ConversationMessage] = []
    mm_tracker = AsyncMultiModalItemTracker(model_config, tokenizer)

    for msg in messages:
664
665
666
667
668
        sub_messages = _parse_chat_message_content(
            msg,
            mm_tracker,
            model_config.chat_template_text_format,
        )
669
670
671

        conversation.extend(sub_messages)

672
673
    _postprocess_messages(conversation)

674
675
676
    return conversation, mm_tracker.all_mm_data()


677
678
def apply_hf_chat_template(
    tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
679
680
681
682
683
    conversation: List[ConversationMessage],
    chat_template: Optional[str],
    *,
    tokenize: bool = False,  # Different from HF's default
    **kwargs: Any,
684
) -> str:
685
686
687
688
689
690
    if chat_template is None and tokenizer.chat_template is None:
        raise ValueError(
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one.")

691
692
693
694
695
696
    return tokenizer.apply_chat_template(
        conversation=conversation,  # type: ignore[arg-type]
        chat_template=chat_template,
        tokenize=tokenize,
        **kwargs,
    )
697
698


699
700
701
def apply_mistral_chat_template(
    tokenizer: MistralTokenizer,
    messages: List[ChatCompletionMessageParam],
702
    chat_template: Optional[str] = None,
703
704
    **kwargs: Any,
) -> List[int]:
705
    if chat_template is not None:
706
        print_warning_once(
707
            "'chat_template' cannot be overridden for mistral tokenizer.")
708
    if "add_generation_prompt" in kwargs:
709
        print_warning_once(
710
711
712
            "'add_generation_prompt' is not supported for mistral tokenizer, "
            "so it will be ignored.")
    if "continue_final_message" in kwargs:
713
        print_warning_once(
714
715
            "'continue_final_message' is not supported for mistral tokenizer, "
            "so it will be ignored.")
716

717
718
    return tokenizer.apply_chat_template(
        messages=messages,
719
720
        **kwargs,
    )