parse.py 15.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
6
from abc import ABC, abstractmethod
from collections import UserDict
from collections.abc import Callable, Iterator, Mapping, Sequence
7
8
9
10
11
12
13
14
15
16
from typing import (
    TYPE_CHECKING,
    Any,
    Generic,
    Literal,
    NamedTuple,
    Optional,
    TypeVar,
    Union,
)
17
18
19
20
21

import numpy as np
import torch
from typing_extensions import TypeAlias, TypeGuard, assert_never

22
from vllm.utils import LazyLoader, is_list_of
23

24
from .audio import AudioResampler
25
26
27
28
29
30
31
32
33
34
35
36
from .inputs import (
    AudioItem,
    HfAudioItem,
    HfImageItem,
    HfVideoItem,
    ImageItem,
    ModalityData,
    MultiModalDataDict,
    MultiModalFieldConfig,
    MultiModalKwargsItems,
    VideoItem,
)
37
38
39
40

_T = TypeVar("_T")
_I = TypeVar("_I")

41
42
43
44
45
if TYPE_CHECKING:
    import PIL.Image as PILImage
else:
    PILImage = LazyLoader("PILImage", globals(), "PIL.Image")

46
47

class ModalityDataItems(ABC, Generic[_T, _I]):
48
    """
49
50
    Represents data items for a modality in
    [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
51
    """
52

53
    def __init__(self, data: _T, modality: str) -> None:
54
55
        super().__init__()

56
        self.data: _T = data
57
58
59
        self.modality = modality

    def __repr__(self) -> str:
60
        return f"{type(self).__name__}(modality={self.modality!r}, len={len(self)})"
61
62
63
64
65
66
67
68
69

    def __len__(self) -> int:
        return self.get_count()

    def __getitem__(self, index: int) -> _I:
        return self.get(index)

    if TYPE_CHECKING:
        # Auto-generated
70
        def __iter__(self) -> Iterator[_I]: ...
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97

    @abstractmethod
    def get_count(self) -> int:
        """Get the number of data items."""
        raise NotImplementedError

    @abstractmethod
    def get(self, index: int) -> _I:
        """Get a data item by its index."""
        raise NotImplementedError

    def get_all(self) -> list[_I]:
        """Get all data items."""
        return [self.get(idx) for idx in range(self.get_count())]

    @abstractmethod
    def get_processor_data(self) -> Mapping[str, object]:
        """Get the data to pass to the HF processor."""
        raise NotImplementedError

    @abstractmethod
    def get_passthrough_data(self) -> Mapping[str, object]:
        """Get the data to pass directly to the model."""
        raise NotImplementedError


class ProcessorBatchItems(ModalityDataItems[Sequence[_T], _T]):
98
    """Base class for data items that are arranged in a list."""
99
100
101
102
103
104
105
106
107
108
109
110
111
112

    def get_count(self) -> int:
        return len(self.data)

    def get(self, index: int) -> _T:
        return self.data[index]

    def get_processor_data(self) -> Mapping[str, object]:
        return {f"{self.modality}s": self.data}

    def get_passthrough_data(self) -> Mapping[str, object]:
        return {}


113
114
115
class EmbeddingItems(
    ModalityDataItems[Union[torch.Tensor, list[torch.Tensor]], torch.Tensor]
):
116
117
118
119
    """
    Base class for data items that are expressed as a batched embedding tensor,
    or a list of embedding tensors (one per item).
    """
120
121
122
123

    def get_count(self) -> int:
        return len(self.data)

124
    def get(self, index: int) -> torch.Tensor:
125
126
127
128
129
130
131
132
        return self.data[index]

    def get_processor_data(self) -> Mapping[str, object]:
        return {}

    def get_passthrough_data(self) -> Mapping[str, object]:
        return {f"{self.modality}_embeds": self.data}

133
134
135
    def get_feature_size(self, item_idx: int) -> int:
        return len(self.get(item_idx))

136

137
138
139
class DictEmbeddingItems(
    ModalityDataItems[Mapping[str, torch.Tensor], Mapping[str, torch.Tensor]]
):
140
141
142
143
144
145
146
147
148
149
150
    """
    Base class for data items that are expressed as a dictionary of tensors.

    Usually, the dictionary keys correspond to the outputs of HF processor.
    """

    def __init__(
        self,
        data: Mapping[str, torch.Tensor],
        modality: str,
        required_fields: set[str],
151
152
153
154
        fields_factory: Callable[
            [Mapping[str, torch.Tensor]],
            Mapping[str, MultiModalFieldConfig],
        ],
155
    ) -> None:
156
157
        from transformers.feature_extraction_utils import BatchFeature

158
159
160
161
162
        super().__init__(data, modality)

        missing_required_data_keys = required_fields - data.keys()
        if missing_required_data_keys:
            data_keys = set(data.keys())
163
164
165
166
            msg = (
                f"The data should contain the fields: {required_fields}, "
                f"but only found the following keys: {data_keys}"
            )
167
168
            raise ValueError(msg)

169
170
171
172
173
174
175
        fields_config = fields_factory(data)
        missing_required_fields = required_fields - fields_config.keys()
        if missing_required_fields:
            fields = set(fields_config.keys())
            msg = f"{required_fields=} should be a subset of {fields=}"
            raise ValueError(msg)

176
177
178
        self.fields_config = fields_config
        self.required_fields = required_fields

179
        self._kwargs = MultiModalKwargsItems.from_hf_inputs(
180
181
182
183
184
            BatchFeature(dict(data)),
            fields_config,
        )

    def get_count(self) -> int:
185
        return len(self._kwargs[self.modality])
186
187

    def get(self, index: int) -> Mapping[str, torch.Tensor]:
188
        return self._kwargs[self.modality][index].get_data()
189
190
191
192
193
194
195
196

    def get_processor_data(self) -> Mapping[str, object]:
        return {}

    def get_passthrough_data(self) -> Mapping[str, object]:
        return self.data


197
class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):
198
199
200
    def __init__(self, data: Optional[Sequence[HfAudioItem]]) -> None:
        if data is None:
            data = [None]
201
202
        super().__init__(data, "audio")

203
204
205
206
    def get_audio_length(self, item_idx: int) -> int:
        audio = self.get(item_idx)
        return len(audio)

207
208

class AudioEmbeddingItems(EmbeddingItems):
209
    def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
210
211
212
213
214
215
216
217
218
        super().__init__(data, "audio")


class ImageSize(NamedTuple):
    width: int
    height: int


class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):
219
220
221
    def __init__(self, data: Optional[Sequence[HfImageItem]]) -> None:
        if data is None:
            data = [None]
222
223
224
225
226
        super().__init__(data, "image")

    def get_image_size(self, item_idx: int) -> ImageSize:
        image = self.get(item_idx)

227
        if isinstance(image, PILImage.Image):
228
229
230
231
232
233
234
235
236
            return ImageSize(*image.size)
        if isinstance(image, (np.ndarray, torch.Tensor)):
            _, h, w = image.shape
            return ImageSize(w, h)

        assert_never(image)


class ImageEmbeddingItems(EmbeddingItems):
237
    def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
238
239
240
241
        super().__init__(data, "image")


class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
242
243
    def __init__(
        self,
244
        data: Optional[Sequence[HfVideoItem]],
245
246
247
        metadata: Optional[
            Union[dict[str, Any], list[Optional[dict[str, Any]]]]
        ] = None,
248
    ) -> None:
249
250
        if data is None:
            data = [None]
251
        super().__init__(data, "video")
252
        self.metadata = metadata
253

254
255
256
257
258
259
    def get_num_frames(self, item_idx: int) -> int:
        return len(self.get(item_idx))

    def get_frame_size(self, item_idx: int) -> ImageSize:
        image = self.get(item_idx)[0]  # Assume that the video isn't empty

260
        if isinstance(image, PILImage.Image):
261
262
263
264
265
266
267
            return ImageSize(*image.size)
        if isinstance(image, (np.ndarray, torch.Tensor)):
            _, h, w = image.shape
            return ImageSize(w, h)

        assert_never(image)

268
269

class VideoEmbeddingItems(EmbeddingItems):
270
    def __init__(self, data: Union[torch.Tensor, list[torch.Tensor]]) -> None:
271
272
273
274
275
276
277
278
        super().__init__(data, "video")


_D = TypeVar("_D", bound=ModalityDataItems[Any, Any])


class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
    """
279
280
    As [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict], but
    normalized such that each entry corresponds to a list.
281
282
283
284
285
    """

    def get_count(self, modality: str, *, strict: bool = True) -> int:
        """
        Get the number of data items belonging to a modality.
286

287
        If `strict=False`, return `0` instead of raising [`KeyError`][]
288
289
290
291
292
        even if the modality is not found.
        """
        if modality not in self:
            if strict:
                available_modalities = set(self.keys())
293
294
295
296
                raise KeyError(
                    f"Modality {modality!r} not found. "
                    f"Available modalities: {available_modalities}"
                )
297
298
299
300
301
302
303
304
305
306
307
308

            return 0

        return self[modality].get_count()

    def get_all_counts(self) -> Mapping[str, int]:
        """Get the number of items belonging to each modality."""
        return {m: items.get_count() for m, items in self.items()}

    def get_items(
        self,
        modality: str,
309
        typ: Union[type[_D], tuple[type[_D], ...]],
310
311
312
313
314
315
316
    ) -> _D:
        """
        Get the data items belonging to a modality,
        requiring that they belong to a certain type.
        """
        if modality not in self:
            available_modalities = set(self.keys())
317
318
319
320
            raise KeyError(
                f"Modality {modality!r} not found. "
                f"Available modalities: {available_modalities}"
            )
321
322
323

        items = self[modality]
        if not isinstance(items, typ):
324
325
326
327
328
            raise TypeError(
                f"Invalid type of data items for {modality=}. "
                f"Expected type: {typ}, but "
                f"found type: {type(items)}"
            )
329

330
        return items  # type: ignore[return-value]
331
332


333
334
335
ModalityDataParser: TypeAlias = Callable[
    [ModalityData[Any]], Optional[ModalityDataItems[Any, Any]]
]
336
337
338
339


class MultiModalDataParser:
    """
340
341
    Parses [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
    into [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
342
343
344
345

    Args:
        target_sr (float, optional): Enables automatic resampling of audio
            items to the model's expected sampling rate.
346
347
    """

348
349
350
351
352
    def __init__(
        self,
        *,
        target_sr: Optional[float] = None,
        audio_resample_method: Literal["librosa", "scipy"] = "librosa",
353
        video_needs_metadata: bool = False,
354
    ) -> None:
355
356
        super().__init__()

357
358
359
360
        self.audio_resampler = AudioResampler(
            target_sr=target_sr,
            method=audio_resample_method,
        )
361
        self.video_needs_metadata = video_needs_metadata
362

363
    def _is_embeddings(
364
        self, data: object
365
    ) -> TypeGuard[Union[torch.Tensor, list[torch.Tensor]]]:
366
367
368
        if isinstance(data, torch.Tensor):
            return data.ndim == 3
        if is_list_of(data, torch.Tensor):
369
370
371
372
373
374
375
376
377
            return data[0].ndim == 2

        return False

    def _is_empty(self, data: object) -> TypeGuard[None]:
        if isinstance(data, list):
            return len(data) == 0
        if isinstance(data, (np.ndarray, torch.Tensor)):
            return data.size == 0
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395

        return False

    def _get_audio_with_sr(
        self,
        audio: AudioItem,
    ) -> tuple[np.ndarray, Optional[float]]:
        if isinstance(audio, tuple):
            return audio
        if isinstance(audio, list):
            return np.array(audio), None
        if isinstance(audio, np.ndarray):
            return audio, None
        if isinstance(audio, torch.Tensor):
            return audio.numpy(), None

        assert_never(audio)

396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
    def _get_video_with_metadata(
        self,
        video: VideoItem,
    ) -> tuple[np.ndarray, Optional[dict[str, Any]]]:
        if isinstance(video, tuple):
            return video
        if isinstance(video, list):
            return np.array(video), None
        if isinstance(video, np.ndarray):
            return video, None
        if isinstance(video, torch.Tensor):
            return video.numpy(), None

        assert_never(video)

411
412
413
    def _parse_audio_data(
        self,
        data: ModalityData[AudioItem],
414
    ) -> Optional[ModalityDataItems[Any, Any]]:
415
416
417
        if data is None:
            return AudioProcessorItems(None)

418
        # also check single audio item with sampling rate
419
420
421
        if self._is_empty(data) or (
            isinstance(data, tuple) and self._is_empty(data[0])
        ):
422
423
            return None

424
425
426
        if self._is_embeddings(data):
            return AudioEmbeddingItems(data)

427
428
429
430
431
432
        if (
            is_list_of(data, float)
            or isinstance(data, (np.ndarray, torch.Tensor))
            and data.ndim == 1
            or isinstance(data, tuple)
        ):
433
434
435
436
437
438
439
440
441
442
443
444
            data_items = [data]
        elif isinstance(data, (np.ndarray, torch.Tensor)):
            data_items = [elem for elem in data]
        else:
            data_items = data

        new_audios = list[np.ndarray]()
        for data_item in data_items:
            audio, orig_sr = self._get_audio_with_sr(data_item)
            if orig_sr is None:
                new_audio = audio
            else:
445
                new_audio = self.audio_resampler.resample(audio, orig_sr=orig_sr)
446
447
448
449
450
451
452
453

            new_audios.append(new_audio)

        return AudioProcessorItems(new_audios)

    def _parse_image_data(
        self,
        data: ModalityData[ImageItem],
454
    ) -> Optional[ModalityDataItems[Any, Any]]:
455
456
457
        if data is None:
            return ImageProcessorItems(None)

458
459
460
        if self._is_empty(data):
            return None

461
462
463
        if self._is_embeddings(data):
            return ImageEmbeddingItems(data)

464
465
466
467
468
        if (
            isinstance(data, PILImage.Image)
            or isinstance(data, (np.ndarray, torch.Tensor))
            and data.ndim == 3
        ):
469
470
471
472
473
474
475
476
477
478
479
            data_items = [data]
        elif isinstance(data, (np.ndarray, torch.Tensor)):
            data_items = [elem for elem in data]
        else:
            data_items = data

        return ImageProcessorItems(data_items)

    def _parse_video_data(
        self,
        data: ModalityData[VideoItem],
480
    ) -> Optional[ModalityDataItems[Any, Any]]:
481
482
483
        if data is None:
            return VideoProcessorItems(None)

484
485
486
        if self._is_empty(data):
            return None

487
488
489
        if self._is_embeddings(data):
            return VideoEmbeddingItems(data)

490
491
492
493
494
        if (
            is_list_of(data, PILImage.Image)
            or isinstance(data, (np.ndarray, torch.Tensor))
            and data.ndim == 4
        ):
495
496
497
            data_items = [data]
        elif isinstance(data, (np.ndarray, torch.Tensor)):
            data_items = [elem for elem in data]
498
499
        elif isinstance(data, tuple) and len(data) == 2:
            data_items = [data]
500
501
502
        else:
            data_items = data

503
504
505
506
507
508
509
510
511
512
513
514
515
516
        new_videos = list[tuple[np.ndarray, Optional[dict[str, Any]]]]()
        metadata_lst: list[Optional[dict[str, Any]]] = []
        for data_item in data_items:
            video, metadata = self._get_video_with_metadata(data_item)
            if self.video_needs_metadata:
                new_videos.append((video, metadata))
                metadata_lst.append(metadata)
            else:
                new_videos.append(video)

        if not self.video_needs_metadata:
            metadata = None

        return VideoProcessorItems(new_videos, metadata=metadata_lst)
517
518
519
520
521
522
523
524

    def _get_subparsers(self) -> Mapping[str, ModalityDataParser]:
        return {
            "audio": self._parse_audio_data,
            "image": self._parse_image_data,
            "video": self._parse_video_data,
        }

525
    def parse_mm_data(self, mm_data: MultiModalDataDict) -> MultiModalDataItems:
526
527
528
529
530
531
532
        subparsers = self._get_subparsers()

        mm_items = MultiModalDataItems()
        for k, v in mm_data.items():
            if k not in subparsers:
                raise ValueError(f"Unsupported modality: {k}")

533
534
535
            # ignore empty embedding data
            if (parsed_data := subparsers[k](v)) is not None:
                mm_items[k] = parsed_data
536
537

        return mm_items