protocol.py 35.6 KB
Newer Older
1
2
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
Zhuohan Li's avatar
Zhuohan Li committed
3
import time
4
from argparse import Namespace
5
from typing import Any, Dict, List, Literal, Optional, Union
Zhuohan Li's avatar
Zhuohan Li committed
6

7
import torch
8
from openai.types.chat import ChatCompletionContentPartParam
9
from pydantic import BaseModel, ConfigDict, Field, model_validator
10
from typing_extensions import Annotated, Required, TypedDict
Zhuohan Li's avatar
Zhuohan Li committed
11

12
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
from vllm.pooling_params import PoolingParams
14
15
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
16
from vllm.sequence import Logprob
17
from vllm.utils import random_uuid
18

19
20
21
# torch is mocked during docs generation,
# so we have to provide the values as literals
_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
22
_LONG_INFO: Union["torch.iinfo", Namespace]
23
24
25
26
27
28
29
30
31
32
33
34
35
36

try:
    from sphinx.ext.autodoc.mock import _MockModule

    if isinstance(torch, _MockModule):
        _LONG_INFO = _MOCK_LONG_INFO
    else:
        _LONG_INFO = torch.iinfo(torch.long)
except ModuleNotFoundError:
    _LONG_INFO = torch.iinfo(torch.long)

assert _LONG_INFO.min == _MOCK_LONG_INFO.min
assert _LONG_INFO.max == _MOCK_LONG_INFO.max

Zhuohan Li's avatar
Zhuohan Li committed
37

38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class CustomChatCompletionMessageParam(TypedDict, total=False):
    """Enables custom roles in the Chat Completion API."""
    role: Required[str]
    """The role of the message's author."""

    content: Union[str, List[ChatCompletionContentPartParam]]
    """The contents of the message."""

    name: str
    """An optional name for the participant.

    Provides the model information to differentiate between participants of the
    same role.
    """

    tool_call_id: Optional[str]

    tool_calls: Optional[List[dict]]


58
59
60
61
62
63
class OpenAIBaseModel(BaseModel):
    # OpenAI API does not allow extra fields
    model_config = ConfigDict(extra="forbid")


class ErrorResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
64
65
66
67
    object: str = "error"
    message: str
    type: str
    param: Optional[str] = None
68
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
69
70


71
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
72
73
74
75
76
77
78
79
80
81
82
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
    group: Optional[str] = None
83
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
84
85


86
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
87
88
89
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
90
    owned_by: str = "vllm"
Zhuohan Li's avatar
Zhuohan Li committed
91
92
    root: Optional[str] = None
    parent: Optional[str] = None
93
    max_model_len: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
94
95
96
    permission: List[ModelPermission] = Field(default_factory=list)


97
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
98
99
100
101
    object: str = "list"
    data: List[ModelCard] = Field(default_factory=list)


102
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
103
104
105
106
107
    prompt_tokens: int = 0
    total_tokens: int = 0
    completion_tokens: Optional[int] = 0


108
109
110
111
112
class RequestResponseMetadata(BaseModel):
    request_id: str
    final_usage_info: Optional[UsageInfo] = None


113
114
115
116
117
118
119
120
121
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
    json_schema: Optional[Dict[str, Any]] = Field(default=None, alias='schema')
    strict: Optional[bool] = None


122
class ResponseFormat(OpenAIBaseModel):
123
124
125
    # type must be "json_schema", "json_object" or "text"
    type: Literal["text", "json_object", "json_schema"]
    json_schema: Optional[JsonSchemaResponseFormat] = None
126
127


128
class StreamOptions(OpenAIBaseModel):
129
    include_usage: Optional[bool] = True
130
    continuous_usage_stats: Optional[bool] = False
131
132


133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
class FunctionDefinition(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    parameters: Optional[Dict[str, Any]] = None


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


153
class ChatCompletionRequest(OpenAIBaseModel):
154
155
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
156
    messages: List[ChatCompletionMessageParam]
157
158
159
160
    model: str
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
    logprobs: Optional[bool] = False
161
    top_logprobs: Optional[int] = 0
162
    max_tokens: Optional[int] = None
163
164
165
    n: Optional[int] = 1
    presence_penalty: Optional[float] = 0.0
    response_format: Optional[ResponseFormat] = None
166
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
167
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
168
    stream: Optional[bool] = False
169
    stream_options: Optional[StreamOptions] = None
170
171
    temperature: Optional[float] = 0.7
    top_p: Optional[float] = 1.0
172
    tools: Optional[List[ChatCompletionToolsParam]] = None
173
    tool_choice: Optional[Union[Literal["none"], Literal["auto"],
174
                                ChatCompletionNamedToolChoiceParam]] = "none"
175
176
177

    # NOTE this will be ignored by VLLM -- the model determines the behavior
    parallel_tool_calls: Optional[bool] = False
Zhuohan Li's avatar
Zhuohan Li committed
178
    user: Optional[str] = None
179
180

    # doc: begin-chat-completion-sampling-params
181
    best_of: Optional[int] = None
182
183
184
185
186
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
187
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
188
189
190
191
192
193
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
194
    prompt_logprobs: Optional[int] = None
195
196
197
    # doc: end-chat-completion-sampling-params

    # doc: begin-chat-completion-extra-params
198
    echo: bool = Field(
199
200
201
202
203
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
            "if they belong to the same role."),
    )
204
    add_generation_prompt: bool = Field(
205
206
207
208
209
210
        default=True,
        description=
        ("If true, the generation prompt will be added to the chat template. "
         "This is a parameter used by chat template in tokenizer config of the "
         "model."),
    )
211
212
213
214
215
216
217
218
219
    continue_final_message: bool = Field(
        default=False,
        description=
        ("If this is set, the chat will be formatted so that the final "
         "message in the chat is open-ended, without any EOS tokens. The "
         "model will continue this message rather than starting a new one. "
         "This allows you to \"prefill\" part of the model's response for it. "
         "Cannot be used at the same time as `add_generation_prompt`."),
    )
220
    add_special_tokens: bool = Field(
221
222
223
224
225
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
226
            "special tokens so this should be set to false (as is the "
227
228
            "default)."),
    )
229
230
231
232
233
234
235
236
237
238
239
240
241
    documents: Optional[List[Dict[str, str]]] = Field(
        default=None,
        description=
        ("A list of dicts representing documents that will be accessible to "
         "the model if it is performing RAG (retrieval-augmented generation)."
         " If the template does not support RAG, this argument will have no "
         "effect. We recommend that each document should be a dict containing "
         "\"title\" and \"text\" keys."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
242
243
244
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."),
245
246
247
248
249
250
    )
    chat_template_kwargs: Optional[Dict[str, Any]] = Field(
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
        description=("If specified, the output will follow the JSON schema."),
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
270
271
272
273
274
275
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be either "
            "'outlines' / 'lm-format-enforcer'"))
276
277
278
279
280
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
281
282
283
284
285
286
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
287
288
289
290
291
292
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."))
293
294

    # doc: end-chat-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
295

296
297
298
299
300
301
302
303
304
305
306
307
308
309
    def to_beam_search_params(self,
                              default_max_tokens: int) -> BeamSearchParams:
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens

        n = self.n if self.n is not None else 1
        temperature = self.temperature if self.temperature is not None else 0.0

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
310
            length_penalty=self.length_penalty,
311
            include_stop_str_in_output=self.include_stop_str_in_output)
312

313
    def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
314
315
316
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens
317

318
319
320
321
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

322
        guided_json_object = None
323
324
325
326
327
328
329
330
331
        if self.response_format is not None:
            if self.response_format.type == "json_object":
                guided_json_object = True
            elif self.response_format.type == "json_schema":
                json_schema = self.response_format.json_schema
                assert json_schema is not None
                self.guided_json = json_schema.json_schema
                if self.guided_decoding_backend is None:
                    self.guided_decoding_backend = "lm-format-enforcer"
332
333
334
335
336
337
338
339
340

        guided_decoding = GuidedDecodingParams.from_optional(
            json=self._get_guided_json_from_tool() or self.guided_json,
            regex=self.guided_regex,
            choice=self.guided_choice,
            grammar=self.guided_grammar,
            json_object=guided_json_object,
            backend=self.guided_decoding_backend,
            whitespace_pattern=self.guided_whitespace_pattern)
341

342
        return SamplingParams.from_optional(
343
            n=self.n,
344
            best_of=self.best_of,
345
346
347
348
349
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
350
            top_k=self.top_k,
351
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
352
            seed=self.seed,
353
354
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
355
            logprobs=self.top_logprobs if self.logprobs else None,
356
            prompt_logprobs=prompt_logprobs,
357
            ignore_eos=self.ignore_eos,
358
            max_tokens=max_tokens,
359
            min_tokens=self.min_tokens,
360
361
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
362
            include_stop_str_in_output=self.include_stop_str_in_output,
363
            truncate_prompt_tokens=self.truncate_prompt_tokens,
364
365
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
            guided_decoding=guided_decoding,
            logit_bias=self.logit_bias)

    def _get_guided_json_from_tool(
            self) -> Optional[Union[str, dict, BaseModel]]:
        # user has chosen to not use any tool
        if self.tool_choice == "none" or self.tools is None:
            return None

        # user has chosen to use a named tool
        if type(self.tool_choice) is ChatCompletionNamedToolChoiceParam:
            tool_name = self.tool_choice.function.name
            tools = {tool.function.name: tool.function for tool in self.tools}
            if tool_name not in tools:
                raise ValueError(
                    f"Tool '{tool_name}' has not been passed in `tools`.")
            tool = tools[tool_name]
            return tool.parameters

        return None
386

387
    @model_validator(mode="before")
388
    @classmethod
389
390
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
391
            raise ValueError(
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
                "Stream options can only be defined when `stream=True`.")

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (top_logprobs := data.get("top_logprobs")) is not None:
            if top_logprobs < 0:
                raise ValueError("`top_logprobs` must be a positive value.")

            if not data.get("logprobs"):
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )

        return data
417

418
419
420
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
421
422
423
        if isinstance(data, ValueError):
            raise data

424
425
426
427
428
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
429
        # you can only use one kind of guided decoding
430
431
432
433
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
434
        # you can only either use guided decoding or tools, not both
435
436
        if guide_count > 1 and data.get("tool_choice",
                                        "none") not in ("none", "auto"):
437
438
439
440
441
442
            raise ValueError(
                "You can only either use guided decoding or tools, not both.")
        return data

    @model_validator(mode="before")
    @classmethod
443
444
445
446
    def check_tool_usage(cls, data):

        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
447
        if "tool_choice" not in data and data.get("tools"):
448
449
450
451
452
453
            data["tool_choice"] = "auto"

        # if "tool_choice" is specified -- validation
        if "tool_choice" in data:

            # ensure that if "tool choice" is specified, tools are present
454
455
456
            if "tools" not in data or data["tools"] is None:
                raise ValueError(
                    "When using `tool_choice`, `tools` must be set.")
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489

            # make sure that tool choice is either a named tool
            # OR that it's set to "auto"
            if data["tool_choice"] != "auto" and not isinstance(
                    data["tool_choice"], dict):
                raise ValueError(
                    "`tool_choice` must either be a named tool or \"auto\". "
                    "`tool_choice=\"none\" is not supported.")

            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
                specified_function = data["tool_choice"]["function"]
                if not specified_function:
                    raise ValueError(
                        "Incorrectly formatted `tool_choice`. Should be like "
                        "`{\"type\": \"function\","
                        " \"function\": {\"name\": \"my_function\"}}`")
                specified_function_name = specified_function["name"]
                if not specified_function_name:
                    raise ValueError(
                        "Incorrectly formatted `tool_choice`. Should be like "
                        "`{\"type\": \"function\", "
                        "\"function\": {\"name\": \"my_function\"}}`")
                for tool in data["tools"]:
                    if tool["function"]["name"] == specified_function_name:
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
                        " of the specified `tools`")
490
491
        return data

492
493
494
495
496
497
498
499
500
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

Zhuohan Li's avatar
Zhuohan Li committed
501

502
class CompletionRequest(OpenAIBaseModel):
503
504
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
Zhuohan Li's avatar
Zhuohan Li committed
505
    model: str
506
    prompt: Union[List[int], List[List[int]], str, List[str]]
507
    best_of: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
508
509
510
    echo: Optional[bool] = False
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
511
512
    logprobs: Optional[int] = None
    max_tokens: Optional[int] = 16
513
    n: int = 1
514
    presence_penalty: Optional[float] = 0.0
515
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
516
517
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
    stream: Optional[bool] = False
518
    stream_options: Optional[StreamOptions] = None
519
520
521
    suffix: Optional[str] = None
    temperature: Optional[float] = 1.0
    top_p: Optional[float] = 1.0
Zhuohan Li's avatar
Zhuohan Li committed
522
    user: Optional[str] = None
523
524

    # doc: begin-completion-sampling-params
525
526
527
528
529
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
530
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
531
532
533
534
535
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
536
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
537
    allowed_token_ids: Optional[List[int]] = None
538
    prompt_logprobs: Optional[int] = None
539
540
541
    # doc: end-completion-sampling-params

    # doc: begin-completion-extra-params
542
543
    add_special_tokens: bool = Field(
        default=True,
544
        description=(
545
546
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."),
547
548
549
550
551
    )
    response_format: Optional[ResponseFormat] = Field(
        default=None,
        description=
        ("Similar to chat completion, this parameter specifies the format of "
552
553
         "output. Only {'type': 'json_object'}, {'type': 'json_schema'} or "
         "{'type': 'text' } is supported."),
554
555
556
    )
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
557
        description="If specified, the output will follow the JSON schema.",
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
574
575
576
577
578
579
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be one of "
            "'outlines' / 'lm-format-enforcer'"))
580
581
582
583
584
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
585
586
587
588
589
590
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
591
592

    # doc: end-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
593

594
595
596
597
598
599
600
601
602
603
604
605
606
607
    def to_beam_search_params(self,
                              default_max_tokens: int) -> BeamSearchParams:
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens

        n = self.n if self.n is not None else 1
        temperature = self.temperature if self.temperature is not None else 0.0

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
608
            length_penalty=self.length_penalty,
609
            include_stop_str_in_output=self.include_stop_str_in_output)
610

611
    def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
612
613
614
615
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens

616
617
618
619
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

620
621
        echo_without_generation = self.echo and self.max_tokens == 0

622
623
624
625
626
627
628
629
630
631
632
633
634
        guided_json_object = None
        if (self.response_format is not None
                and self.response_format.type == "json_object"):
            guided_json_object = True

        guided_decoding = GuidedDecodingParams.from_optional(
            json=self.guided_json,
            regex=self.guided_regex,
            choice=self.guided_choice,
            grammar=self.guided_grammar,
            json_object=guided_json_object,
            backend=self.guided_decoding_backend,
            whitespace_pattern=self.guided_whitespace_pattern)
635

636
        return SamplingParams.from_optional(
637
638
639
640
641
642
643
644
645
            n=self.n,
            best_of=self.best_of,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
            top_k=self.top_k,
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
646
            seed=self.seed,
647
648
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
649
            logprobs=self.logprobs,
650
            ignore_eos=self.ignore_eos,
651
            max_tokens=max_tokens if not echo_without_generation else 1,
652
            min_tokens=self.min_tokens,
653
            prompt_logprobs=prompt_logprobs,
654
            skip_special_tokens=self.skip_special_tokens,
655
            spaces_between_special_tokens=self.spaces_between_special_tokens,
656
            include_stop_str_in_output=self.include_stop_str_in_output,
657
            truncate_prompt_tokens=self.truncate_prompt_tokens,
658
659
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
660
661
662
            guided_decoding=guided_decoding,
            logit_bias=self.logit_bias,
            allowed_token_ids=self.allowed_token_ids)
663

664
665
666
667
668
669
670
671
672
673
674
675
676
677
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
        return data

678
679
680
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
681
682
683
684
685
686
687
688
689
690
691
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
            raise ValueError("`logprobs` must be a positive value.")

692
693
        return data

694
695
696
697
698
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
            raise ValueError(
699
700
                "Stream options can only be defined when `stream=True`.")

701
702
        return data

Zhuohan Li's avatar
Zhuohan Li committed
703

704
class EmbeddingRequest(OpenAIBaseModel):
705
706
707
708
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
    model: str
    input: Union[List[int], List[List[int]], str, List[str]]
709
    encoding_format: Literal["float", "base64"] = "float"
710
711
    dimensions: Optional[int] = None
    user: Optional[str] = None
712
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
713
714
715
716
717
718

    # doc: begin-embedding-pooling-params
    additional_data: Optional[Any] = None

    # doc: end-embedding-pooling-params

719
720
721
722
723
724
725
726
727
728
    # doc: begin-embedding-extra-params
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))

    # doc: end-embedding-extra-params

729
730
731
732
    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


733
class CompletionLogProbs(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
734
735
736
    text_offset: List[int] = Field(default_factory=list)
    token_logprobs: List[Optional[float]] = Field(default_factory=list)
    tokens: List[str] = Field(default_factory=list)
737
738
    top_logprobs: List[Optional[Dict[str,
                                     float]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
739
740


741
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
742
743
    index: int
    text: str
744
    logprobs: Optional[CompletionLogProbs] = None
745
746
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
747
748
749
750
751
752
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
753
    prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
Zhuohan Li's avatar
Zhuohan Li committed
754
755


756
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
757
758
759
760
761
762
763
764
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseChoice]
    usage: UsageInfo


765
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
766
767
    index: int
    text: str
768
    logprobs: Optional[CompletionLogProbs] = None
769
770
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
771
772
773
774
775
776
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
Zhuohan Li's avatar
Zhuohan Li committed
777
778


779
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
780
781
782
783
784
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseStreamChoice]
785
    usage: Optional[UsageInfo] = Field(default=None)
786
787


788
class EmbeddingResponseData(OpenAIBaseModel):
789
790
    index: int
    object: str = "embedding"
791
    embedding: Union[List[float], str]
792
793


794
class EmbeddingResponse(OpenAIBaseModel):
795
796
797
798
799
800
801
802
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    data: List[EmbeddingResponseData]
    usage: UsageInfo


803
804
805
806
807
808
809
810
811
812
813
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    function: FunctionCall


814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
class DeltaFunctionCall(BaseModel):
    name: Optional[str] = None
    arguments: Optional[str] = None


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    index: int
    function: Optional[DeltaFunctionCall] = None


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
    tool_calls: List[ToolCall]

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
    content: Optional[str] = None


839
class ChatMessage(OpenAIBaseModel):
840
    role: str
841
    content: Optional[str] = None
842
    tool_calls: List[ToolCall] = Field(default_factory=list)
843
844


845
846
847
848
849
850
851
852
853
854
855
856
857
858
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
    bytes: Optional[List[int]] = None


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
    top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list)


class ChatCompletionLogProbs(OpenAIBaseModel):
    content: Optional[List[ChatCompletionLogProbsContent]] = None


859
class ChatCompletionResponseChoice(OpenAIBaseModel):
860
861
    index: int
    message: ChatMessage
862
    logprobs: Optional[ChatCompletionLogProbs] = None
863
864
865
    # per OpenAI spec this is the default
    finish_reason: Optional[str] = "stop"
    # not part of the OpenAI spec but included in vLLM for legacy reasons
866
    stop_reason: Optional[Union[int, str]] = None
867
868


869
class ChatCompletionResponse(OpenAIBaseModel):
870
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
871
    object: Literal["chat.completion"] = "chat.completion"
872
873
874
875
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseChoice]
    usage: UsageInfo
876
    prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
877
878


879
class DeltaMessage(OpenAIBaseModel):
880
881
    role: Optional[str] = None
    content: Optional[str] = None
882
    tool_calls: List[DeltaToolCall] = Field(default_factory=list)
883
884


885
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
886
887
    index: int
    delta: DeltaMessage
888
    logprobs: Optional[ChatCompletionLogProbs] = None
889
    finish_reason: Optional[str] = None
890
    stop_reason: Optional[Union[int, str]] = None
891
892


893
class ChatCompletionStreamResponse(OpenAIBaseModel):
894
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
895
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
896
897
898
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseStreamChoice]
899
    usage: Optional[UsageInfo] = Field(default=None)
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920


class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

921
    # The parameters of the request.
922
    body: Union[ChatCompletionRequest, EmbeddingRequest]
923
924


925
926
927
928
929
930
931
932
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
933
    body: Optional[Union[ChatCompletionResponse, EmbeddingResponse]] = None
934
935


936
937
938
939
940
941
942
943
944
945
946
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

947
    response: Optional[BatchResponseData]
948
949
950
951

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
    error: Optional[Any]
952
953


954
955
956
957
958
959
960
961
962
963
964
class TokenizeCompletionRequest(OpenAIBaseModel):
    model: str
    prompt: str

    add_special_tokens: bool = Field(default=True)


class TokenizeChatRequest(OpenAIBaseModel):
    model: str
    messages: List[ChatCompletionMessageParam]

965
    add_generation_prompt: bool = Field(default=True)
966
    continue_final_message: bool = Field(default=False)
967
    add_special_tokens: bool = Field(default=False)
968

969
970
971
972
973
974
975
976
977
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

978
979

TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
980
981
982
983
984


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
985
    tokens: List[int]
986
987
988
989
990
991
992
993
994


class DetokenizeRequest(OpenAIBaseModel):
    model: str
    tokens: List[int]


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
995
996
997
998
999
1000
1001
1002
1003
1004


class LoadLoraAdapterRequest(BaseModel):
    lora_name: str
    lora_path: str


class UnloadLoraAdapterRequest(BaseModel):
    lora_name: str
    lora_int_id: Optional[int] = Field(default=None)