protocol.py 30.9 KB
Newer Older
1
2
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
Zhuohan Li's avatar
Zhuohan Li committed
3
import time
4
from argparse import Namespace
5
from typing import Any, Dict, List, Literal, Optional, Union
Zhuohan Li's avatar
Zhuohan Li committed
6

7
import torch
8
from openai.types.chat import ChatCompletionContentPartParam
9
from pydantic import BaseModel, ConfigDict, Field, model_validator
10
from typing_extensions import Annotated, Required, TypedDict
Zhuohan Li's avatar
Zhuohan Li committed
11

12
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
from vllm.entrypoints.openai.logits_processors import get_logits_processors
14
from vllm.pooling_params import PoolingParams
15
16
from vllm.sampling_params import (LogitsProcessor, RequestOutputKind,
                                  SamplingParams)
17
from vllm.sequence import Logprob
18
from vllm.transformers_utils.tokenizer import AnyTokenizer
19
from vllm.utils import random_uuid
20

21
22
23
# torch is mocked during docs generation,
# so we have to provide the values as literals
_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
24
_LONG_INFO: Union["torch.iinfo", Namespace]
25
26
27
28
29
30
31
32
33
34
35
36
37
38

try:
    from sphinx.ext.autodoc.mock import _MockModule

    if isinstance(torch, _MockModule):
        _LONG_INFO = _MOCK_LONG_INFO
    else:
        _LONG_INFO = torch.iinfo(torch.long)
except ModuleNotFoundError:
    _LONG_INFO = torch.iinfo(torch.long)

assert _LONG_INFO.min == _MOCK_LONG_INFO.min
assert _LONG_INFO.max == _MOCK_LONG_INFO.max

Zhuohan Li's avatar
Zhuohan Li committed
39

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
class CustomChatCompletionMessageParam(TypedDict, total=False):
    """Enables custom roles in the Chat Completion API."""
    role: Required[str]
    """The role of the message's author."""

    content: Union[str, List[ChatCompletionContentPartParam]]
    """The contents of the message."""

    name: str
    """An optional name for the participant.

    Provides the model information to differentiate between participants of the
    same role.
    """

    tool_call_id: Optional[str]

    tool_calls: Optional[List[dict]]


60
61
62
63
64
65
class OpenAIBaseModel(BaseModel):
    # OpenAI API does not allow extra fields
    model_config = ConfigDict(extra="forbid")


class ErrorResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
66
67
68
69
    object: str = "error"
    message: str
    type: str
    param: Optional[str] = None
70
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
71
72


73
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
74
75
76
77
78
79
80
81
82
83
84
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
    group: Optional[str] = None
85
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
86
87


88
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
89
90
91
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
92
    owned_by: str = "vllm"
Zhuohan Li's avatar
Zhuohan Li committed
93
94
    root: Optional[str] = None
    parent: Optional[str] = None
95
    max_model_len: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
96
97
98
    permission: List[ModelPermission] = Field(default_factory=list)


99
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
100
101
102
103
    object: str = "list"
    data: List[ModelCard] = Field(default_factory=list)


104
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
105
106
107
108
109
    prompt_tokens: int = 0
    total_tokens: int = 0
    completion_tokens: Optional[int] = 0


110
111
112
113
114
class RequestResponseMetadata(BaseModel):
    request_id: str
    final_usage_info: Optional[UsageInfo] = None


115
116
117
118
119
120
121
122
123
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
    json_schema: Optional[Dict[str, Any]] = Field(default=None, alias='schema')
    strict: Optional[bool] = None


124
class ResponseFormat(OpenAIBaseModel):
125
126
127
    # type must be "json_schema", "json_object" or "text"
    type: Literal["text", "json_object", "json_schema"]
    json_schema: Optional[JsonSchemaResponseFormat] = None
128
129


130
class StreamOptions(OpenAIBaseModel):
131
132
    include_usage: Optional[bool] = True
    continuous_usage_stats: Optional[bool] = True
133
134


135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
class FunctionDefinition(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    parameters: Optional[Dict[str, Any]] = None


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


155
class ChatCompletionRequest(OpenAIBaseModel):
156
157
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
158
    messages: List[ChatCompletionMessageParam]
159
160
161
162
    model: str
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
    logprobs: Optional[bool] = False
163
    top_logprobs: Optional[int] = 0
164
    max_tokens: Optional[int] = None
165
166
167
    n: Optional[int] = 1
    presence_penalty: Optional[float] = 0.0
    response_format: Optional[ResponseFormat] = None
168
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
169
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
170
    stream: Optional[bool] = False
171
    stream_options: Optional[StreamOptions] = None
172
173
    temperature: Optional[float] = 0.7
    top_p: Optional[float] = 1.0
174
    tools: Optional[List[ChatCompletionToolsParam]] = None
175
    tool_choice: Optional[Union[Literal["none"], Literal["auto"],
176
                                ChatCompletionNamedToolChoiceParam]] = "none"
177
178
179

    # NOTE this will be ignored by VLLM -- the model determines the behavior
    parallel_tool_calls: Optional[bool] = False
Zhuohan Li's avatar
Zhuohan Li committed
180
    user: Optional[str] = None
181
182

    # doc: begin-chat-completion-sampling-params
183
    best_of: Optional[int] = None
184
185
186
187
188
189
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
    early_stopping: bool = False
190
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
191
192
193
194
195
196
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
197
    prompt_logprobs: Optional[int] = None
198
199
200
    # doc: end-chat-completion-sampling-params

    # doc: begin-chat-completion-extra-params
201
    echo: bool = Field(
202
203
204
205
206
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
            "if they belong to the same role."),
    )
207
    add_generation_prompt: bool = Field(
208
209
210
211
212
213
        default=True,
        description=
        ("If true, the generation prompt will be added to the chat template. "
         "This is a parameter used by chat template in tokenizer config of the "
         "model."),
    )
214
    add_special_tokens: bool = Field(
215
216
217
218
219
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
220
            "special tokens so this should be set to false (as is the "
221
222
            "default)."),
    )
223
224
225
226
227
228
229
230
231
232
233
234
235
    documents: Optional[List[Dict[str, str]]] = Field(
        default=None,
        description=
        ("A list of dicts representing documents that will be accessible to "
         "the model if it is performing RAG (retrieval-augmented generation)."
         " If the template does not support RAG, this argument will have no "
         "effect. We recommend that each document should be a dict containing "
         "\"title\" and \"text\" keys."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
236
237
238
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."),
239
240
241
242
243
244
    )
    chat_template_kwargs: Optional[Dict[str, Any]] = Field(
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
        description=("If specified, the output will follow the JSON schema."),
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
264
265
266
267
268
269
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be either "
            "'outlines' / 'lm-format-enforcer'"))
270
271
272
273
274
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
275
276

    # doc: end-chat-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
277

278
    def to_sampling_params(
279
            self, tokenizer: AnyTokenizer,
280
281
282
283
284
            guided_decode_logits_processor: Optional[LogitsProcessor],
            default_max_tokens: int) -> SamplingParams:
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens
285

286
287
288
289
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

290
        # We now allow logprobs being true without top_logrobs.
291
292
293
294
295
        logits_processors = get_logits_processors(
            logit_bias=self.logit_bias,
            allowed_token_ids=None,
            tokenizer=tokenizer,
        )
296
297
        if guided_decode_logits_processor:
            logits_processors.append(guided_decode_logits_processor)
298

299
        return SamplingParams.from_optional(
300
            n=self.n,
301
            best_of=self.best_of,
302
303
304
305
306
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
307
            top_k=self.top_k,
308
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
309
            seed=self.seed,
310
311
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
312
            logprobs=self.top_logprobs if self.logprobs else None,
313
            prompt_logprobs=prompt_logprobs,
314
            ignore_eos=self.ignore_eos,
315
            max_tokens=max_tokens,
316
            min_tokens=self.min_tokens,
317
            use_beam_search=self.use_beam_search,
318
            early_stopping=self.early_stopping,
319
320
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
321
322
            include_stop_str_in_output=self.include_stop_str_in_output,
            length_penalty=self.length_penalty,
323
            logits_processors=logits_processors,
324
            truncate_prompt_tokens=self.truncate_prompt_tokens,
325
326
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
327
328
        )

329
    @model_validator(mode="before")
330
    @classmethod
331
332
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
333
            raise ValueError(
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
                "Stream options can only be defined when `stream=True`.")

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (top_logprobs := data.get("top_logprobs")) is not None:
            if top_logprobs < 0:
                raise ValueError("`top_logprobs` must be a positive value.")

            if not data.get("logprobs"):
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )

        return data
359

360
361
362
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
363
364
365
        if isinstance(data, ValueError):
            raise data

366
367
368
369
370
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
371
        # you can only use one kind of guided decoding
372
373
374
375
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
376
        # you can only either use guided decoding or tools, not both
377
378
        if guide_count > 1 and data.get("tool_choice",
                                        "none") not in ("none", "auto"):
379
380
381
382
383
384
            raise ValueError(
                "You can only either use guided decoding or tools, not both.")
        return data

    @model_validator(mode="before")
    @classmethod
385
386
387
388
389
390
391
392
393
394
395
    def check_tool_usage(cls, data):

        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
        if "tool_choice" not in data and "tools" in data:
            data["tool_choice"] = "auto"

        # if "tool_choice" is specified -- validation
        if "tool_choice" in data:

            # ensure that if "tool choice" is specified, tools are present
396
397
398
            if "tools" not in data or data["tools"] is None:
                raise ValueError(
                    "When using `tool_choice`, `tools` must be set.")
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431

            # make sure that tool choice is either a named tool
            # OR that it's set to "auto"
            if data["tool_choice"] != "auto" and not isinstance(
                    data["tool_choice"], dict):
                raise ValueError(
                    "`tool_choice` must either be a named tool or \"auto\". "
                    "`tool_choice=\"none\" is not supported.")

            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
                specified_function = data["tool_choice"]["function"]
                if not specified_function:
                    raise ValueError(
                        "Incorrectly formatted `tool_choice`. Should be like "
                        "`{\"type\": \"function\","
                        " \"function\": {\"name\": \"my_function\"}}`")
                specified_function_name = specified_function["name"]
                if not specified_function_name:
                    raise ValueError(
                        "Incorrectly formatted `tool_choice`. Should be like "
                        "`{\"type\": \"function\", "
                        "\"function\": {\"name\": \"my_function\"}}`")
                for tool in data["tools"]:
                    if tool["function"]["name"] == specified_function_name:
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
                        " of the specified `tools`")
432
433
        return data

Zhuohan Li's avatar
Zhuohan Li committed
434

435
class CompletionRequest(OpenAIBaseModel):
436
437
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
Zhuohan Li's avatar
Zhuohan Li committed
438
    model: str
439
    prompt: Union[List[int], List[List[int]], str, List[str]]
440
    best_of: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
441
442
443
    echo: Optional[bool] = False
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
444
445
    logprobs: Optional[int] = None
    max_tokens: Optional[int] = 16
446
    n: int = 1
447
    presence_penalty: Optional[float] = 0.0
448
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
449
450
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
    stream: Optional[bool] = False
451
    stream_options: Optional[StreamOptions] = None
452
453
454
    suffix: Optional[str] = None
    temperature: Optional[float] = 1.0
    top_p: Optional[float] = 1.0
Zhuohan Li's avatar
Zhuohan Li committed
455
    user: Optional[str] = None
456
457

    # doc: begin-completion-sampling-params
458
459
460
461
462
463
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
    early_stopping: bool = False
464
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
465
466
467
468
469
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
470
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
471
    allowed_token_ids: Optional[List[int]] = None
472
    prompt_logprobs: Optional[int] = None
473
474
475
    # doc: end-completion-sampling-params

    # doc: begin-completion-extra-params
476
477
    add_special_tokens: bool = Field(
        default=True,
478
        description=(
479
480
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."),
481
482
483
484
485
486
487
488
489
490
    )
    response_format: Optional[ResponseFormat] = Field(
        default=None,
        description=
        ("Similar to chat completion, this parameter specifies the format of "
         "output. Only {'type': 'json_object'} or {'type': 'text' } is "
         "supported."),
    )
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
491
        description="If specified, the output will follow the JSON schema.",
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
508
509
510
511
512
513
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be one of "
            "'outlines' / 'lm-format-enforcer'"))
514
515
516
517
518
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
519
520

    # doc: end-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
521

522
    def to_sampling_params(
523
            self, tokenizer: AnyTokenizer,
524
525
526
527
528
529
            guided_decode_logits_processor: Optional[LogitsProcessor],
            default_max_tokens: int) -> SamplingParams:
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens

530
531
532
533
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

534
535
        echo_without_generation = self.echo and self.max_tokens == 0

536
537
538
539
540
        logits_processors = get_logits_processors(
            logit_bias=self.logit_bias,
            allowed_token_ids=self.allowed_token_ids,
            tokenizer=tokenizer,
        )
541
542
        if guided_decode_logits_processor:
            logits_processors.append(guided_decode_logits_processor)
543

544
        return SamplingParams.from_optional(
545
546
547
548
549
550
551
552
553
            n=self.n,
            best_of=self.best_of,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
            top_k=self.top_k,
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
554
            seed=self.seed,
555
556
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
557
            logprobs=self.logprobs,
558
            ignore_eos=self.ignore_eos,
559
            max_tokens=max_tokens if not echo_without_generation else 1,
560
            min_tokens=self.min_tokens,
561
            use_beam_search=self.use_beam_search,
562
            early_stopping=self.early_stopping,
563
            prompt_logprobs=prompt_logprobs,
564
            skip_special_tokens=self.skip_special_tokens,
565
            spaces_between_special_tokens=self.spaces_between_special_tokens,
566
567
            include_stop_str_in_output=self.include_stop_str_in_output,
            length_penalty=self.length_penalty,
568
            logits_processors=logits_processors,
569
            truncate_prompt_tokens=self.truncate_prompt_tokens,
570
571
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
572
573
        )

574
575
576
577
578
579
580
581
582
583
584
585
586
587
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
        return data

588
589
590
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
591
592
593
594
595
596
597
598
599
600
601
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
            raise ValueError("`logprobs` must be a positive value.")

602
603
        return data

604
605
606
607
608
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
            raise ValueError(
609
610
                "Stream options can only be defined when `stream=True`.")

611
612
        return data

Zhuohan Li's avatar
Zhuohan Li committed
613

614
class EmbeddingRequest(OpenAIBaseModel):
615
616
617
618
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
    model: str
    input: Union[List[int], List[List[int]], str, List[str]]
619
    encoding_format: Literal["float", "base64"] = "float"
620
621
622
623
624
625
626
627
628
629
630
631
    dimensions: Optional[int] = None
    user: Optional[str] = None

    # doc: begin-embedding-pooling-params
    additional_data: Optional[Any] = None

    # doc: end-embedding-pooling-params

    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


632
class CompletionLogProbs(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
633
634
635
    text_offset: List[int] = Field(default_factory=list)
    token_logprobs: List[Optional[float]] = Field(default_factory=list)
    tokens: List[str] = Field(default_factory=list)
636
637
    top_logprobs: List[Optional[Dict[str,
                                     float]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
638
639


640
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
641
642
    index: int
    text: str
643
    logprobs: Optional[CompletionLogProbs] = None
644
645
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
646
647
648
649
650
651
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
652
    prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
Zhuohan Li's avatar
Zhuohan Li committed
653
654


655
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
656
657
658
659
660
661
662
663
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseChoice]
    usage: UsageInfo


664
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
665
666
    index: int
    text: str
667
    logprobs: Optional[CompletionLogProbs] = None
668
669
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
670
671
672
673
674
675
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
Zhuohan Li's avatar
Zhuohan Li committed
676
677


678
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
679
680
681
682
683
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseStreamChoice]
684
    usage: Optional[UsageInfo] = Field(default=None)
685
686


687
class EmbeddingResponseData(OpenAIBaseModel):
688
689
    index: int
    object: str = "embedding"
690
    embedding: Union[List[float], str]
691
692


693
class EmbeddingResponse(OpenAIBaseModel):
694
695
696
697
698
699
700
701
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    data: List[EmbeddingResponseData]
    usage: UsageInfo


702
703
704
705
706
707
708
709
710
711
712
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    function: FunctionCall


713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
class DeltaFunctionCall(BaseModel):
    name: Optional[str] = None
    arguments: Optional[str] = None


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    index: int
    function: Optional[DeltaFunctionCall] = None


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
    tool_calls: List[ToolCall]

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
    content: Optional[str] = None


738
class ChatMessage(OpenAIBaseModel):
739
    role: str
740
    content: Optional[str] = None
741
    tool_calls: List[ToolCall] = Field(default_factory=list)
742
743


744
745
746
747
748
749
750
751
752
753
754
755
756
757
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
    bytes: Optional[List[int]] = None


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
    top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list)


class ChatCompletionLogProbs(OpenAIBaseModel):
    content: Optional[List[ChatCompletionLogProbsContent]] = None


758
class ChatCompletionResponseChoice(OpenAIBaseModel):
759
760
    index: int
    message: ChatMessage
761
    logprobs: Optional[ChatCompletionLogProbs] = None
762
763
764
    # per OpenAI spec this is the default
    finish_reason: Optional[str] = "stop"
    # not part of the OpenAI spec but included in vLLM for legacy reasons
765
    stop_reason: Optional[Union[int, str]] = None
766
767


768
class ChatCompletionResponse(OpenAIBaseModel):
769
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
770
    object: Literal["chat.completion"] = "chat.completion"
771
772
773
774
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseChoice]
    usage: UsageInfo
775
    prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
776
777


778
class DeltaMessage(OpenAIBaseModel):
779
780
    role: Optional[str] = None
    content: Optional[str] = None
781
    tool_calls: List[DeltaToolCall] = Field(default_factory=list)
782
783


784
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
785
786
    index: int
    delta: DeltaMessage
787
    logprobs: Optional[ChatCompletionLogProbs] = None
788
    finish_reason: Optional[str] = None
789
    stop_reason: Optional[Union[int, str]] = None
790
791


792
class ChatCompletionStreamResponse(OpenAIBaseModel):
793
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
794
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
795
796
797
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseStreamChoice]
798
    usage: Optional[UsageInfo] = Field(default=None)
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819


class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

820
    # The parameters of the request.
821
    body: Union[ChatCompletionRequest, EmbeddingRequest]
822
823


824
825
826
827
828
829
830
831
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
832
    body: Optional[Union[ChatCompletionResponse, EmbeddingResponse]] = None
833
834


835
836
837
838
839
840
841
842
843
844
845
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

846
    response: Optional[BatchResponseData]
847
848
849
850

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
    error: Optional[Any]
851
852


853
854
855
856
857
858
859
860
861
862
863
class TokenizeCompletionRequest(OpenAIBaseModel):
    model: str
    prompt: str

    add_special_tokens: bool = Field(default=True)


class TokenizeChatRequest(OpenAIBaseModel):
    model: str
    messages: List[ChatCompletionMessageParam]

864
865
    add_generation_prompt: bool = Field(default=True)
    add_special_tokens: bool = Field(default=False)
866
867
868


TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
869
870
871
872
873


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
874
    tokens: List[int]
875
876
877
878
879
880
881
882
883


class DetokenizeRequest(OpenAIBaseModel):
    model: str
    tokens: List[int]


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
884
885
886
887
888
889
890
891
892
893


class LoadLoraAdapterRequest(BaseModel):
    lora_name: str
    lora_path: str


class UnloadLoraAdapterRequest(BaseModel):
    lora_name: str
    lora_int_id: Optional[int] = Field(default=None)