data.py 11.3 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Generic, Literal, Optional, Union, cast
5

6
import torch
7
from typing_extensions import NotRequired, TypedDict, TypeIs, TypeVar
8
9

if TYPE_CHECKING:
10
11
    from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalInputs,
                                        MultiModalUUIDDict)
12
13
14
15
16
17
18
19


class TextPrompt(TypedDict):
    """Schema for a text prompt."""

    prompt: str
    """The input text to be tokenized before passing to the model."""

20
    multi_modal_data: NotRequired["MultiModalDataDict"]
21
22
23
24
25
    """
    Optional multi-modal data to pass to the model,
    if the model supports it.
    """

26
    mm_processor_kwargs: NotRequired[dict[str, Any]]
27
28
29
30
31
32
33
    """
    Optional multi-modal processor kwargs to be forwarded to the
    multimodal input mapper & processor. Note that if multiple modalities
    have registered mappers etc for the model being considered, we attempt
    to pass the mm_processor_kwargs to each of them.
    """

34
35
36
37
38
39
40
41
42
    multi_modal_uuids: NotRequired["MultiModalUUIDDict"]
    """
    Optional user-specified UUIDs for multimodal items, mapped by modality.
    Lists must match the number of items per modality and may contain `None`.
    For `None` entries, the hasher will compute IDs automatically; non-None
    entries override the default hashes for caching, and MUST be unique per
    multimodal item.
    """

43
44
45
46
47
    cache_salt: NotRequired[str]
    """
    Optional cache salt to be used for prefix caching.
    """

48
49
50
51

class TokensPrompt(TypedDict):
    """Schema for a tokenized prompt."""

52
    prompt_token_ids: list[int]
53
54
    """A list of token IDs to pass to the model."""

55
    token_type_ids: NotRequired[list[int]]
56
57
    """A list of token type IDs to pass to the cross encoder model."""

58
    multi_modal_data: NotRequired["MultiModalDataDict"]
59
    """
60
    Optional multi-modal data to pass to the model,
61
62
63
    if the model supports it.
    """

64
    mm_processor_kwargs: NotRequired[dict[str, Any]]
65
    """
66
    Optional multi-modal processor kwargs to be forwarded to the
67
68
69
70
71
    multimodal input mapper & processor. Note that if multiple modalities
    have registered mappers etc for the model being considered, we attempt
    to pass the mm_processor_kwargs to each of them.
    """

72
73
74
75
76
77
78
79
    multi_modal_uuids: NotRequired["MultiModalUUIDDict"]
    """
    Optional user-specified UUIDs for multimodal items, mapped by modality.
    Lists must match the number of items per modality and may contain `None`.
    For `None` entries, the hasher will compute IDs automatically; non-None
    entries override the default hashes for caching.
    """

80
81
82
83
84
    cache_salt: NotRequired[str]
    """
    Optional cache salt to be used for prefix caching.
    """

85

86
87
88
89
90
91
class EmbedsPrompt(TypedDict):
    """Schema for a prompt provided via token embeddings."""

    prompt_embeds: torch.Tensor
    """The embeddings of the prompt."""

92
93
94
95
96
    cache_salt: NotRequired[str]
    """
    Optional cache salt to be used for prefix caching.
    """

97

98
99
100
101
102
103
104
105
106
107
class DataPrompt(TypedDict):
    """Represents generic inputs handled by IO processor plugins."""

    data: Any
    """The input data"""

    data_format: str
    """The input data format"""


108
SingletonPrompt = Union[str, TextPrompt, TokensPrompt, EmbedsPrompt]
109
"""
110
Set of possible schemas for a single prompt:
111

112
113
114
- A text prompt ([`str`][] or [`TextPrompt`][vllm.inputs.data.TextPrompt])
- A tokenized prompt ([`TokensPrompt`][vllm.inputs.data.TokensPrompt])
- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])
115
116
117
118
119

Note that "singleton" is as opposed to a data structure
which encapsulates multiple prompts, i.e. of the sort
which may be utilized for encoder/decoder models when
the user desires to express both the encoder & decoder
120
121
prompts explicitly, i.e. 
[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
122

123
124
A prompt of type [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] may be 
employed as (1) input to a decoder-only model, (2) input to
125
126
127
the encoder of an encoder/decoder model, in the scenario
where the decoder-prompt is not specified explicitly, or
(3) as a member of a larger data structure encapsulating
128
129
more than one prompt, i.e. 
[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
130
131
"""

132
133
134
135
136
137
138
139
140
141
142

def is_tokens_prompt(prompt: SingletonPrompt) -> TypeIs[TokensPrompt]:
    return (isinstance(prompt, dict) and "prompt_token_ids" in prompt
            and "prompt_embeds" not in prompt)


def is_embeds_prompt(prompt: SingletonPrompt) -> TypeIs[EmbedsPrompt]:
    return (isinstance(prompt, dict) and "prompt_token_ids" not in prompt
            and "prompt_embeds" in prompt)


143
_T1_co = TypeVar("_T1_co",
144
145
                 bound=SingletonPrompt,
                 default=SingletonPrompt,
146
147
                 covariant=True)
_T2_co = TypeVar("_T2_co",
148
149
                 bound=SingletonPrompt,
                 default=SingletonPrompt,
150
                 covariant=True)
151

152
153
154

# TODO: Make fields ReadOnly once mypy supports it
class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
155
156
157
    """
    Represents an encoder/decoder model input prompt,
    comprising an explicit encoder prompt and a decoder prompt.
158

159
    The encoder and decoder prompts, respectively, may be formatted
160
161
    according to any of the
    [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] schemas,
162
    and are not required to have the same schema.
163

164
165
166
    Only the encoder prompt may have multi-modal data. mm_processor_kwargs
    should be at the top-level, and should not be set in the encoder/decoder
    prompts, since they are agnostic to the encoder/decoder.
167

168
169
170
    Note that an
    [`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
    may not be used as an input to a decoder-only model,
171
    and that the `encoder_prompt` and `decoder_prompt`
172
    fields of this data structure themselves must be
173
    [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] instances.
174
175
    """

176
    encoder_prompt: _T1_co
177

178
    decoder_prompt: Optional[_T2_co]
179

180
    mm_processor_kwargs: NotRequired[dict[str, Any]]
181

182

183
PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
184
185
186
187
"""
Set of possible schemas for an LLM input, including
both decoder-only and encoder/decoder input types:

188
189
190
- A text prompt ([`str`][] or [`TextPrompt`][vllm.inputs.data.TextPrompt])
- A tokenized prompt ([`TokensPrompt`][vllm.inputs.data.TokensPrompt])
- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])
191
- A single data structure containing both an encoder and a decoder prompt
192
  ([`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt])
193
194
195
"""


196
197
class TokenInputs(TypedDict):
    """Represents token-based inputs."""
198
199
200
201

    type: Literal["token"]
    """The type of inputs."""

202
    prompt_token_ids: list[int]
203
204
    """The token IDs of the prompt."""

205
    prompt: NotRequired[str]
206
207
208
209
    """
    The original prompt text corresponding to the token IDs, if available.
    """

210
211
212
213
214
    cache_salt: NotRequired[str]
    """
    Optional cache salt to be used for prefix caching.
    """

215

216
def token_inputs(
217
    prompt_token_ids: list[int],
218
    prompt: Optional[str] = None,
219
    cache_salt: Optional[str] = None,
220
) -> TokenInputs:
221
222
    """Construct [`TokenInputs`][vllm.inputs.data.TokenInputs] from optional
    values."""
223
    inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)
224
225
226

    if prompt is not None:
        inputs["prompt"] = prompt
227
228
    if cache_salt is not None:
        inputs["cache_salt"] = cache_salt
229
230
231
232

    return inputs


233
234
235
236
237
238
239
240
241
class EmbedsInputs(TypedDict):
    """Represents embeddings-based inputs."""

    type: Literal["embeds"]
    """The type of inputs."""

    prompt_embeds: torch.Tensor
    """The embeddings of the prompt."""

242
243
244
245
246
    cache_salt: NotRequired[str]
    """
    Optional cache salt to be used for prefix caching.
    """

247

248
249
250
251
def embeds_inputs(
    prompt_embeds: torch.Tensor,
    cache_salt: Optional[str] = None,
) -> EmbedsInputs:
252
253
    """Construct [`EmbedsInputs`][vllm.inputs.data.EmbedsInputs] from optional
    values."""
254
255
256
257
    inputs = EmbedsInputs(type="embeds", prompt_embeds=prompt_embeds)

    if cache_salt is not None:
        inputs["cache_salt"] = cache_salt
258
259
260
261
262

    return inputs


DecoderOnlyInputs = Union[TokenInputs, EmbedsInputs, "MultiModalInputs"]
263
"""
264
The inputs in [`LLMEngine`][vllm.engine.llm_engine.LLMEngine] before they are
265
266
267
268
269
passed to the model executor.
This specifies the data required for decoder-only models.
"""


270
class EncoderDecoderInputs(TypedDict):
271
    """
272
273
    The inputs in [`LLMEngine`][vllm.engine.llm_engine.LLMEngine] before they
    are passed to the model executor.
274
275
276

    This specifies the required data for encoder-decoder models.
    """
277

278
    encoder: Union[TokenInputs, "MultiModalInputs"]
279
    """The inputs for the encoder portion."""
280

281
    decoder: Union[TokenInputs, "MultiModalInputs"]
282
    """The inputs for the decoder portion."""
283

284

285
SingletonInputs = Union[TokenInputs, EmbedsInputs, "MultiModalInputs"]
286
"""
287
288
A processed [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] which can be 
passed to [`vllm.sequence.Sequence`][].
289
290
291
292
"""

ProcessorInputs = Union[DecoderOnlyInputs, EncoderDecoderInputs]
"""
293
The outputs from [`vllm.inputs.preprocess.InputPreprocessor`][].
294
"""
295

296
297
_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
_T2 = TypeVar("_T2", bound=SingletonPrompt, default=SingletonPrompt)
298
299


300
301
302
def build_explicit_enc_dec_prompt(
    encoder_prompt: _T1,
    decoder_prompt: Optional[_T2],
303
    mm_processor_kwargs: Optional[dict[str, Any]] = None,
304
) -> ExplicitEncoderDecoderPrompt[_T1, _T2]:
305
306
307
308
309
    if mm_processor_kwargs is None:
        mm_processor_kwargs = {}
    return ExplicitEncoderDecoderPrompt(
        encoder_prompt=encoder_prompt,
        decoder_prompt=decoder_prompt,
310
311
        mm_processor_kwargs=mm_processor_kwargs,
    )
312
313
314
315
316


def zip_enc_dec_prompts(
    enc_prompts: Iterable[_T1],
    dec_prompts: Iterable[Optional[_T2]],
317
318
319
    mm_processor_kwargs: Optional[Union[Iterable[dict[str, Any]],
                                        dict[str, Any]]] = None,
) -> list[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
320
    """
321
    Zip encoder and decoder prompts together into a list of
322
323
    [`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
    instances.
324

325
326
327
    ``mm_processor_kwargs`` may also be provided; if a dict is passed, the same
    dictionary will be used for every encoder/decoder prompt. If an iterable is
    provided, it will be zipped with the encoder/decoder prompts.
328
329
    """
    if mm_processor_kwargs is None:
330
        mm_processor_kwargs = cast(dict[str, Any], {})
331
    if isinstance(mm_processor_kwargs, dict):
332
        return [
333
            build_explicit_enc_dec_prompt(
334
335
336
337
338
                encoder_prompt,
                decoder_prompt,
                cast(dict[str, Any], mm_processor_kwargs),
            ) for (encoder_prompt,
                   decoder_prompt) in zip(enc_prompts, dec_prompts)
339
        ]
340
    return [
341
342
343
344
        build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt,
                                      mm_proc_kwargs)
        for (encoder_prompt, decoder_prompt, mm_proc_kwargs
             ) in zip(enc_prompts, dec_prompts, mm_processor_kwargs)
345
346
    ]

347

348
349
def to_enc_dec_tuple_list(
    enc_dec_prompts: Iterable[ExplicitEncoderDecoderPrompt[_T1, _T2]],
350
) -> list[tuple[_T1, Optional[_T2]]]:
351
352
353
    return [(enc_dec_prompt["encoder_prompt"],
             enc_dec_prompt["decoder_prompt"])
            for enc_dec_prompt in enc_dec_prompts]