protocol.py 36.1 KB
Newer Older
1
2
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
Zhuohan Li's avatar
Zhuohan Li committed
3
import time
4
from argparse import Namespace
5
from typing import Any, Dict, List, Literal, Optional, Union
Zhuohan Li's avatar
Zhuohan Li committed
6

7
import torch
8
from openai.types.chat import ChatCompletionContentPartParam
9
from pydantic import BaseModel, ConfigDict, Field, model_validator
10
from typing_extensions import Annotated, Required, TypedDict
Zhuohan Li's avatar
Zhuohan Li committed
11

12
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
from vllm.pooling_params import PoolingParams
14
15
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
16
from vllm.sequence import Logprob
17
from vllm.utils import random_uuid
18

19
20
21
# torch is mocked during docs generation,
# so we have to provide the values as literals
_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
22
_LONG_INFO: Union["torch.iinfo", Namespace]
23
24
25
26
27
28
29
30
31
32
33
34
35
36

try:
    from sphinx.ext.autodoc.mock import _MockModule

    if isinstance(torch, _MockModule):
        _LONG_INFO = _MOCK_LONG_INFO
    else:
        _LONG_INFO = torch.iinfo(torch.long)
except ModuleNotFoundError:
    _LONG_INFO = torch.iinfo(torch.long)

assert _LONG_INFO.min == _MOCK_LONG_INFO.min
assert _LONG_INFO.max == _MOCK_LONG_INFO.max

Zhuohan Li's avatar
Zhuohan Li committed
37

38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class CustomChatCompletionMessageParam(TypedDict, total=False):
    """Enables custom roles in the Chat Completion API."""
    role: Required[str]
    """The role of the message's author."""

    content: Union[str, List[ChatCompletionContentPartParam]]
    """The contents of the message."""

    name: str
    """An optional name for the participant.

    Provides the model information to differentiate between participants of the
    same role.
    """

    tool_call_id: Optional[str]

    tool_calls: Optional[List[dict]]


58
59
60
61
62
63
class OpenAIBaseModel(BaseModel):
    # OpenAI API does not allow extra fields
    model_config = ConfigDict(extra="forbid")


class ErrorResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
64
65
66
67
    object: str = "error"
    message: str
    type: str
    param: Optional[str] = None
68
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
69
70


71
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
72
73
74
75
76
77
78
79
80
81
82
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
    group: Optional[str] = None
83
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
84
85


86
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
87
88
89
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
90
    owned_by: str = "vllm"
Zhuohan Li's avatar
Zhuohan Li committed
91
92
    root: Optional[str] = None
    parent: Optional[str] = None
93
    max_model_len: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
94
95
96
    permission: List[ModelPermission] = Field(default_factory=list)


97
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
98
99
100
101
    object: str = "list"
    data: List[ModelCard] = Field(default_factory=list)


102
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
103
104
105
106
107
    prompt_tokens: int = 0
    total_tokens: int = 0
    completion_tokens: Optional[int] = 0


108
109
110
111
112
class RequestResponseMetadata(BaseModel):
    request_id: str
    final_usage_info: Optional[UsageInfo] = None


113
114
115
116
117
118
119
120
121
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
    json_schema: Optional[Dict[str, Any]] = Field(default=None, alias='schema')
    strict: Optional[bool] = None


122
class ResponseFormat(OpenAIBaseModel):
123
124
125
    # type must be "json_schema", "json_object" or "text"
    type: Literal["text", "json_object", "json_schema"]
    json_schema: Optional[JsonSchemaResponseFormat] = None
126
127


128
class StreamOptions(OpenAIBaseModel):
129
    include_usage: Optional[bool] = True
130
    continuous_usage_stats: Optional[bool] = False
131
132


133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
class FunctionDefinition(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    parameters: Optional[Dict[str, Any]] = None


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


153
class ChatCompletionRequest(OpenAIBaseModel):
154
155
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
156
    messages: List[ChatCompletionMessageParam]
157
158
159
160
    model: str
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
    logprobs: Optional[bool] = False
161
    top_logprobs: Optional[int] = 0
162
163
164
165
166
167
    # TODO(#9845): remove max_tokens when field is removed from OpenAI API
    max_tokens: Optional[int] = Field(
        default=None,
        deprecated=
        'max_tokens is deprecated in favor of the max_completion_tokens field')
    max_completion_tokens: Optional[int] = None
168
169
170
    n: Optional[int] = 1
    presence_penalty: Optional[float] = 0.0
    response_format: Optional[ResponseFormat] = None
171
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
172
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
173
    stream: Optional[bool] = False
174
    stream_options: Optional[StreamOptions] = None
175
176
    temperature: Optional[float] = 0.7
    top_p: Optional[float] = 1.0
177
    tools: Optional[List[ChatCompletionToolsParam]] = None
178
    tool_choice: Optional[Union[Literal["none"], Literal["auto"],
179
                                ChatCompletionNamedToolChoiceParam]] = "none"
180
181
182

    # NOTE this will be ignored by VLLM -- the model determines the behavior
    parallel_tool_calls: Optional[bool] = False
Zhuohan Li's avatar
Zhuohan Li committed
183
    user: Optional[str] = None
184
185

    # doc: begin-chat-completion-sampling-params
186
    best_of: Optional[int] = None
187
188
189
190
191
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
192
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
193
194
195
196
197
198
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
199
    prompt_logprobs: Optional[int] = None
200
201
202
    # doc: end-chat-completion-sampling-params

    # doc: begin-chat-completion-extra-params
203
    echo: bool = Field(
204
205
206
207
208
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
            "if they belong to the same role."),
    )
209
    add_generation_prompt: bool = Field(
210
211
212
213
214
215
        default=True,
        description=
        ("If true, the generation prompt will be added to the chat template. "
         "This is a parameter used by chat template in tokenizer config of the "
         "model."),
    )
216
217
218
219
220
221
222
223
224
    continue_final_message: bool = Field(
        default=False,
        description=
        ("If this is set, the chat will be formatted so that the final "
         "message in the chat is open-ended, without any EOS tokens. The "
         "model will continue this message rather than starting a new one. "
         "This allows you to \"prefill\" part of the model's response for it. "
         "Cannot be used at the same time as `add_generation_prompt`."),
    )
225
    add_special_tokens: bool = Field(
226
227
228
229
230
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
231
            "special tokens so this should be set to false (as is the "
232
233
            "default)."),
    )
234
235
236
237
238
239
240
241
242
243
244
245
246
    documents: Optional[List[Dict[str, str]]] = Field(
        default=None,
        description=
        ("A list of dicts representing documents that will be accessible to "
         "the model if it is performing RAG (retrieval-augmented generation)."
         " If the template does not support RAG, this argument will have no "
         "effect. We recommend that each document should be a dict containing "
         "\"title\" and \"text\" keys."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
247
248
249
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."),
250
251
252
253
254
255
    )
    chat_template_kwargs: Optional[Dict[str, Any]] = Field(
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
        description=("If specified, the output will follow the JSON schema."),
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
275
276
277
278
279
280
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be either "
            "'outlines' / 'lm-format-enforcer'"))
281
282
283
284
285
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
286
287
288
289
290
291
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
292
293
294
295
296
297
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."))
298
299

    # doc: end-chat-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
300

301
302
    def to_beam_search_params(self,
                              default_max_tokens: int) -> BeamSearchParams:
303
304
        # TODO(#9845): remove max_tokens when field is removed from OpenAI API
        max_tokens = self.max_completion_tokens or self.max_tokens
305
306
307
308
309
310
311
312
313
314
315
        if max_tokens is None:
            max_tokens = default_max_tokens

        n = self.n if self.n is not None else 1
        temperature = self.temperature if self.temperature is not None else 0.0

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
316
            length_penalty=self.length_penalty,
317
            include_stop_str_in_output=self.include_stop_str_in_output)
318

319
    def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
320
321
        # TODO(#9845): remove max_tokens when field is removed from OpenAI API
        max_tokens = self.max_completion_tokens or self.max_tokens
322
323
        if max_tokens is None:
            max_tokens = default_max_tokens
324

325
326
327
328
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

329
        guided_json_object = None
330
331
332
333
334
335
336
337
338
        if self.response_format is not None:
            if self.response_format.type == "json_object":
                guided_json_object = True
            elif self.response_format.type == "json_schema":
                json_schema = self.response_format.json_schema
                assert json_schema is not None
                self.guided_json = json_schema.json_schema
                if self.guided_decoding_backend is None:
                    self.guided_decoding_backend = "lm-format-enforcer"
339
340
341
342
343
344
345
346
347

        guided_decoding = GuidedDecodingParams.from_optional(
            json=self._get_guided_json_from_tool() or self.guided_json,
            regex=self.guided_regex,
            choice=self.guided_choice,
            grammar=self.guided_grammar,
            json_object=guided_json_object,
            backend=self.guided_decoding_backend,
            whitespace_pattern=self.guided_whitespace_pattern)
348

349
        return SamplingParams.from_optional(
350
            n=self.n,
351
            best_of=self.best_of,
352
353
354
355
356
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
357
            top_k=self.top_k,
358
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
359
            seed=self.seed,
360
361
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
362
            logprobs=self.top_logprobs if self.logprobs else None,
363
            prompt_logprobs=prompt_logprobs,
364
            ignore_eos=self.ignore_eos,
365
            max_tokens=max_tokens,
366
            min_tokens=self.min_tokens,
367
368
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
369
            include_stop_str_in_output=self.include_stop_str_in_output,
370
            truncate_prompt_tokens=self.truncate_prompt_tokens,
371
372
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
            guided_decoding=guided_decoding,
            logit_bias=self.logit_bias)

    def _get_guided_json_from_tool(
            self) -> Optional[Union[str, dict, BaseModel]]:
        # user has chosen to not use any tool
        if self.tool_choice == "none" or self.tools is None:
            return None

        # user has chosen to use a named tool
        if type(self.tool_choice) is ChatCompletionNamedToolChoiceParam:
            tool_name = self.tool_choice.function.name
            tools = {tool.function.name: tool.function for tool in self.tools}
            if tool_name not in tools:
                raise ValueError(
                    f"Tool '{tool_name}' has not been passed in `tools`.")
            tool = tools[tool_name]
            return tool.parameters

        return None
393

394
    @model_validator(mode="before")
395
    @classmethod
396
397
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
398
            raise ValueError(
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
                "Stream options can only be defined when `stream=True`.")

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (top_logprobs := data.get("top_logprobs")) is not None:
            if top_logprobs < 0:
                raise ValueError("`top_logprobs` must be a positive value.")

            if not data.get("logprobs"):
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )

        return data
424

425
426
427
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
428
429
430
        if isinstance(data, ValueError):
            raise data

431
432
433
434
435
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
436
        # you can only use one kind of guided decoding
437
438
439
440
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
441
        # you can only either use guided decoding or tools, not both
442
443
        if guide_count > 1 and data.get("tool_choice",
                                        "none") not in ("none", "auto"):
444
445
446
447
448
449
            raise ValueError(
                "You can only either use guided decoding or tools, not both.")
        return data

    @model_validator(mode="before")
    @classmethod
450
451
452
453
    def check_tool_usage(cls, data):

        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
454
        if "tool_choice" not in data and data.get("tools"):
455
456
457
458
459
460
            data["tool_choice"] = "auto"

        # if "tool_choice" is specified -- validation
        if "tool_choice" in data:

            # ensure that if "tool choice" is specified, tools are present
461
462
463
            if "tools" not in data or data["tools"] is None:
                raise ValueError(
                    "When using `tool_choice`, `tools` must be set.")
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496

            # make sure that tool choice is either a named tool
            # OR that it's set to "auto"
            if data["tool_choice"] != "auto" and not isinstance(
                    data["tool_choice"], dict):
                raise ValueError(
                    "`tool_choice` must either be a named tool or \"auto\". "
                    "`tool_choice=\"none\" is not supported.")

            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
                specified_function = data["tool_choice"]["function"]
                if not specified_function:
                    raise ValueError(
                        "Incorrectly formatted `tool_choice`. Should be like "
                        "`{\"type\": \"function\","
                        " \"function\": {\"name\": \"my_function\"}}`")
                specified_function_name = specified_function["name"]
                if not specified_function_name:
                    raise ValueError(
                        "Incorrectly formatted `tool_choice`. Should be like "
                        "`{\"type\": \"function\", "
                        "\"function\": {\"name\": \"my_function\"}}`")
                for tool in data["tools"]:
                    if tool["function"]["name"] == specified_function_name:
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
                        " of the specified `tools`")
497
498
        return data

499
500
501
502
503
504
505
506
507
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

Zhuohan Li's avatar
Zhuohan Li committed
508

509
class CompletionRequest(OpenAIBaseModel):
510
511
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
Zhuohan Li's avatar
Zhuohan Li committed
512
    model: str
513
    prompt: Union[List[int], List[List[int]], str, List[str]]
514
    best_of: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
515
516
517
    echo: Optional[bool] = False
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
518
519
    logprobs: Optional[int] = None
    max_tokens: Optional[int] = 16
520
    n: int = 1
521
    presence_penalty: Optional[float] = 0.0
522
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
523
524
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
    stream: Optional[bool] = False
525
    stream_options: Optional[StreamOptions] = None
526
527
528
    suffix: Optional[str] = None
    temperature: Optional[float] = 1.0
    top_p: Optional[float] = 1.0
Zhuohan Li's avatar
Zhuohan Li committed
529
    user: Optional[str] = None
530
531

    # doc: begin-completion-sampling-params
532
533
534
535
536
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
537
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
538
539
540
541
542
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
543
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
544
    allowed_token_ids: Optional[List[int]] = None
545
    prompt_logprobs: Optional[int] = None
546
547
548
    # doc: end-completion-sampling-params

    # doc: begin-completion-extra-params
549
550
    add_special_tokens: bool = Field(
        default=True,
551
        description=(
552
553
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."),
554
555
556
557
558
    )
    response_format: Optional[ResponseFormat] = Field(
        default=None,
        description=
        ("Similar to chat completion, this parameter specifies the format of "
559
560
         "output. Only {'type': 'json_object'}, {'type': 'json_schema'} or "
         "{'type': 'text' } is supported."),
561
562
563
    )
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
564
        description="If specified, the output will follow the JSON schema.",
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
581
582
583
584
585
586
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be one of "
            "'outlines' / 'lm-format-enforcer'"))
587
588
589
590
591
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
592
593
594
595
596
597
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
598
599

    # doc: end-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
600

601
602
603
604
605
606
607
608
609
610
611
612
613
614
    def to_beam_search_params(self,
                              default_max_tokens: int) -> BeamSearchParams:
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens

        n = self.n if self.n is not None else 1
        temperature = self.temperature if self.temperature is not None else 0.0

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
615
            length_penalty=self.length_penalty,
616
            include_stop_str_in_output=self.include_stop_str_in_output)
617

618
    def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
619
620
621
622
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens

623
624
625
626
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

627
628
        echo_without_generation = self.echo and self.max_tokens == 0

629
630
631
632
633
634
635
636
637
638
639
640
641
        guided_json_object = None
        if (self.response_format is not None
                and self.response_format.type == "json_object"):
            guided_json_object = True

        guided_decoding = GuidedDecodingParams.from_optional(
            json=self.guided_json,
            regex=self.guided_regex,
            choice=self.guided_choice,
            grammar=self.guided_grammar,
            json_object=guided_json_object,
            backend=self.guided_decoding_backend,
            whitespace_pattern=self.guided_whitespace_pattern)
642

643
        return SamplingParams.from_optional(
644
645
646
647
648
649
650
651
652
            n=self.n,
            best_of=self.best_of,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
            top_k=self.top_k,
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
653
            seed=self.seed,
654
655
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
656
            logprobs=self.logprobs,
657
            ignore_eos=self.ignore_eos,
658
            max_tokens=max_tokens if not echo_without_generation else 1,
659
            min_tokens=self.min_tokens,
660
            prompt_logprobs=prompt_logprobs,
661
            skip_special_tokens=self.skip_special_tokens,
662
            spaces_between_special_tokens=self.spaces_between_special_tokens,
663
            include_stop_str_in_output=self.include_stop_str_in_output,
664
            truncate_prompt_tokens=self.truncate_prompt_tokens,
665
666
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
667
668
669
            guided_decoding=guided_decoding,
            logit_bias=self.logit_bias,
            allowed_token_ids=self.allowed_token_ids)
670

671
672
673
674
675
676
677
678
679
680
681
682
683
684
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
        return data

685
686
687
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
688
689
690
691
692
693
694
695
696
697
698
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
            raise ValueError("`logprobs` must be a positive value.")

699
700
        return data

701
702
703
704
705
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
            raise ValueError(
706
707
                "Stream options can only be defined when `stream=True`.")

708
709
        return data

Zhuohan Li's avatar
Zhuohan Li committed
710

711
class EmbeddingRequest(OpenAIBaseModel):
712
713
714
715
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
    model: str
    input: Union[List[int], List[List[int]], str, List[str]]
716
    encoding_format: Literal["float", "base64"] = "float"
717
718
    dimensions: Optional[int] = None
    user: Optional[str] = None
719
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
720
721
722
723
724
725

    # doc: begin-embedding-pooling-params
    additional_data: Optional[Any] = None

    # doc: end-embedding-pooling-params

726
727
728
729
730
731
732
733
734
735
    # doc: begin-embedding-extra-params
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))

    # doc: end-embedding-extra-params

736
737
738
739
    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


740
class CompletionLogProbs(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
741
742
743
    text_offset: List[int] = Field(default_factory=list)
    token_logprobs: List[Optional[float]] = Field(default_factory=list)
    tokens: List[str] = Field(default_factory=list)
744
745
    top_logprobs: List[Optional[Dict[str,
                                     float]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
746
747


748
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
749
750
    index: int
    text: str
751
    logprobs: Optional[CompletionLogProbs] = None
752
753
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
754
755
756
757
758
759
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
760
    prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
Zhuohan Li's avatar
Zhuohan Li committed
761
762


763
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
764
765
766
767
768
769
770
771
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseChoice]
    usage: UsageInfo


772
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
773
774
    index: int
    text: str
775
    logprobs: Optional[CompletionLogProbs] = None
776
777
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
778
779
780
781
782
783
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
Zhuohan Li's avatar
Zhuohan Li committed
784
785


786
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
787
788
789
790
791
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseStreamChoice]
792
    usage: Optional[UsageInfo] = Field(default=None)
793
794


795
class EmbeddingResponseData(OpenAIBaseModel):
796
797
    index: int
    object: str = "embedding"
798
    embedding: Union[List[float], str]
799
800


801
class EmbeddingResponse(OpenAIBaseModel):
802
803
804
805
806
807
808
809
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    data: List[EmbeddingResponseData]
    usage: UsageInfo


810
811
812
813
814
815
816
817
818
819
820
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    function: FunctionCall


821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
class DeltaFunctionCall(BaseModel):
    name: Optional[str] = None
    arguments: Optional[str] = None


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    index: int
    function: Optional[DeltaFunctionCall] = None


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
    tool_calls: List[ToolCall]

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
    content: Optional[str] = None


846
class ChatMessage(OpenAIBaseModel):
847
    role: str
848
    content: Optional[str] = None
849
    tool_calls: List[ToolCall] = Field(default_factory=list)
850
851


852
853
854
855
856
857
858
859
860
861
862
863
864
865
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
    bytes: Optional[List[int]] = None


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
    top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list)


class ChatCompletionLogProbs(OpenAIBaseModel):
    content: Optional[List[ChatCompletionLogProbsContent]] = None


866
class ChatCompletionResponseChoice(OpenAIBaseModel):
867
868
    index: int
    message: ChatMessage
869
    logprobs: Optional[ChatCompletionLogProbs] = None
870
871
872
    # per OpenAI spec this is the default
    finish_reason: Optional[str] = "stop"
    # not part of the OpenAI spec but included in vLLM for legacy reasons
873
    stop_reason: Optional[Union[int, str]] = None
874
875


876
class ChatCompletionResponse(OpenAIBaseModel):
877
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
878
    object: Literal["chat.completion"] = "chat.completion"
879
880
881
882
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseChoice]
    usage: UsageInfo
883
    prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
884
885


886
class DeltaMessage(OpenAIBaseModel):
887
888
    role: Optional[str] = None
    content: Optional[str] = None
889
    tool_calls: List[DeltaToolCall] = Field(default_factory=list)
890
891


892
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
893
894
    index: int
    delta: DeltaMessage
895
    logprobs: Optional[ChatCompletionLogProbs] = None
896
    finish_reason: Optional[str] = None
897
    stop_reason: Optional[Union[int, str]] = None
898
899


900
class ChatCompletionStreamResponse(OpenAIBaseModel):
901
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
902
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
903
904
905
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseStreamChoice]
906
    usage: Optional[UsageInfo] = Field(default=None)
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927


class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

928
    # The parameters of the request.
929
    body: Union[ChatCompletionRequest, EmbeddingRequest]
930
931


932
933
934
935
936
937
938
939
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
940
    body: Optional[Union[ChatCompletionResponse, EmbeddingResponse]] = None
941
942


943
944
945
946
947
948
949
950
951
952
953
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

954
    response: Optional[BatchResponseData]
955
956
957
958

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
    error: Optional[Any]
959
960


961
962
963
964
965
966
967
968
969
970
971
class TokenizeCompletionRequest(OpenAIBaseModel):
    model: str
    prompt: str

    add_special_tokens: bool = Field(default=True)


class TokenizeChatRequest(OpenAIBaseModel):
    model: str
    messages: List[ChatCompletionMessageParam]

972
    add_generation_prompt: bool = Field(default=True)
973
    continue_final_message: bool = Field(default=False)
974
    add_special_tokens: bool = Field(default=False)
975

976
977
978
979
980
981
982
983
984
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

985
986

TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
987
988
989
990
991


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
992
    tokens: List[int]
993
994
995
996
997
998
999
1000
1001


class DetokenizeRequest(OpenAIBaseModel):
    model: str
    tokens: List[int]


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011


class LoadLoraAdapterRequest(BaseModel):
    lora_name: str
    lora_path: str


class UnloadLoraAdapterRequest(BaseModel):
    lora_name: str
    lora_int_id: Optional[int] = Field(default=None)