protocol.py 39.7 KB
Newer Older
1
2
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
Zhuohan Li's avatar
Zhuohan Li committed
3
import time
4
from argparse import Namespace
5
from typing import Any, Dict, List, Literal, Optional, Union
Zhuohan Li's avatar
Zhuohan Li committed
6

7
import torch
8
from openai.types.chat import ChatCompletionContentPartParam
9
from pydantic import BaseModel, ConfigDict, Field, model_validator
10
from typing_extensions import Annotated, Required, TypedDict
Zhuohan Li's avatar
Zhuohan Li committed
11

12
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
from vllm.pooling_params import PoolingParams
14
15
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
16
from vllm.sequence import Logprob
17
from vllm.utils import random_uuid
18

19
20
21
# torch is mocked during docs generation,
# so we have to provide the values as literals
_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
22
_LONG_INFO: Union["torch.iinfo", Namespace]
23
24
25
26
27
28
29
30
31
32
33
34
35
36

try:
    from sphinx.ext.autodoc.mock import _MockModule

    if isinstance(torch, _MockModule):
        _LONG_INFO = _MOCK_LONG_INFO
    else:
        _LONG_INFO = torch.iinfo(torch.long)
except ModuleNotFoundError:
    _LONG_INFO = torch.iinfo(torch.long)

assert _LONG_INFO.min == _MOCK_LONG_INFO.min
assert _LONG_INFO.max == _MOCK_LONG_INFO.max

Zhuohan Li's avatar
Zhuohan Li committed
37

38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
class CustomChatCompletionMessageParam(TypedDict, total=False):
    """Enables custom roles in the Chat Completion API."""
    role: Required[str]
    """The role of the message's author."""

    content: Union[str, List[ChatCompletionContentPartParam]]
    """The contents of the message."""

    name: str
    """An optional name for the participant.

    Provides the model information to differentiate between participants of the
    same role.
    """

    tool_call_id: Optional[str]

    tool_calls: Optional[List[dict]]


58
59
60
61
62
63
class OpenAIBaseModel(BaseModel):
    # OpenAI API does not allow extra fields
    model_config = ConfigDict(extra="forbid")


class ErrorResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
64
65
66
67
    object: str = "error"
    message: str
    type: str
    param: Optional[str] = None
68
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
69
70


71
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
72
73
74
75
76
77
78
79
80
81
82
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
    group: Optional[str] = None
83
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
84
85


86
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
87
88
89
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
90
    owned_by: str = "vllm"
Zhuohan Li's avatar
Zhuohan Li committed
91
92
    root: Optional[str] = None
    parent: Optional[str] = None
93
    max_model_len: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
94
95
96
    permission: List[ModelPermission] = Field(default_factory=list)


97
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
98
99
100
101
    object: str = "list"
    data: List[ModelCard] = Field(default_factory=list)


102
103
104
105
class PromptTokenUsageInfo(OpenAIBaseModel):
    cached_tokens: Optional[int] = None


106
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
107
108
109
    prompt_tokens: int = 0
    total_tokens: int = 0
    completion_tokens: Optional[int] = 0
110
    prompt_tokens_details: Optional[PromptTokenUsageInfo] = None
Zhuohan Li's avatar
Zhuohan Li committed
111
112


113
114
115
116
117
class RequestResponseMetadata(BaseModel):
    request_id: str
    final_usage_info: Optional[UsageInfo] = None


118
119
120
121
122
123
124
125
126
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
    json_schema: Optional[Dict[str, Any]] = Field(default=None, alias='schema')
    strict: Optional[bool] = None


127
class ResponseFormat(OpenAIBaseModel):
128
129
130
    # type must be "json_schema", "json_object" or "text"
    type: Literal["text", "json_object", "json_schema"]
    json_schema: Optional[JsonSchemaResponseFormat] = None
131
132


133
class StreamOptions(OpenAIBaseModel):
134
    include_usage: Optional[bool] = True
135
    continuous_usage_stats: Optional[bool] = False
136
137


138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
class FunctionDefinition(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    parameters: Optional[Dict[str, Any]] = None


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


158
class ChatCompletionRequest(OpenAIBaseModel):
159
160
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
161
    messages: List[ChatCompletionMessageParam]
162
163
164
165
    model: str
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
    logprobs: Optional[bool] = False
166
    top_logprobs: Optional[int] = 0
167
168
169
170
171
172
    # TODO(#9845): remove max_tokens when field is removed from OpenAI API
    max_tokens: Optional[int] = Field(
        default=None,
        deprecated=
        'max_tokens is deprecated in favor of the max_completion_tokens field')
    max_completion_tokens: Optional[int] = None
173
174
175
    n: Optional[int] = 1
    presence_penalty: Optional[float] = 0.0
    response_format: Optional[ResponseFormat] = None
176
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
177
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
178
    stream: Optional[bool] = False
179
    stream_options: Optional[StreamOptions] = None
180
181
    temperature: Optional[float] = 0.7
    top_p: Optional[float] = 1.0
182
    tools: Optional[List[ChatCompletionToolsParam]] = None
183
    tool_choice: Optional[Union[Literal["none"], Literal["auto"],
184
                                ChatCompletionNamedToolChoiceParam]] = "none"
185
186
187

    # NOTE this will be ignored by VLLM -- the model determines the behavior
    parallel_tool_calls: Optional[bool] = False
Zhuohan Li's avatar
Zhuohan Li committed
188
    user: Optional[str] = None
189
190

    # doc: begin-chat-completion-sampling-params
191
    best_of: Optional[int] = None
192
193
194
195
196
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
197
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
198
199
200
201
202
203
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
204
    prompt_logprobs: Optional[int] = None
205
206
207
    # doc: end-chat-completion-sampling-params

    # doc: begin-chat-completion-extra-params
208
    echo: bool = Field(
209
210
211
212
213
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
            "if they belong to the same role."),
    )
214
    add_generation_prompt: bool = Field(
215
216
217
218
219
220
        default=True,
        description=
        ("If true, the generation prompt will be added to the chat template. "
         "This is a parameter used by chat template in tokenizer config of the "
         "model."),
    )
221
222
223
224
225
226
227
228
229
    continue_final_message: bool = Field(
        default=False,
        description=
        ("If this is set, the chat will be formatted so that the final "
         "message in the chat is open-ended, without any EOS tokens. The "
         "model will continue this message rather than starting a new one. "
         "This allows you to \"prefill\" part of the model's response for it. "
         "Cannot be used at the same time as `add_generation_prompt`."),
    )
230
    add_special_tokens: bool = Field(
231
232
233
234
235
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
236
            "special tokens so this should be set to false (as is the "
237
238
            "default)."),
    )
239
240
241
242
243
244
245
246
247
248
249
250
251
    documents: Optional[List[Dict[str, str]]] = Field(
        default=None,
        description=
        ("A list of dicts representing documents that will be accessible to "
         "the model if it is performing RAG (retrieval-augmented generation)."
         " If the template does not support RAG, this argument will have no "
         "effect. We recommend that each document should be a dict containing "
         "\"title\" and \"text\" keys."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
252
253
254
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."),
255
256
257
258
259
260
    )
    chat_template_kwargs: Optional[Dict[str, Any]] = Field(
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
        description=("If specified, the output will follow the JSON schema."),
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
280
281
282
283
284
285
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be either "
            "'outlines' / 'lm-format-enforcer'"))
286
287
288
289
290
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
291
292
293
294
295
296
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
297
298
299
300
301
302
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."))
303
304

    # doc: end-chat-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
305

306
307
    def to_beam_search_params(self,
                              default_max_tokens: int) -> BeamSearchParams:
308
309
        # TODO(#9845): remove max_tokens when field is removed from OpenAI API
        max_tokens = self.max_completion_tokens or self.max_tokens
310
311
312
313
314
315
316
317
318
319
320
        if max_tokens is None:
            max_tokens = default_max_tokens

        n = self.n if self.n is not None else 1
        temperature = self.temperature if self.temperature is not None else 0.0

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
321
            length_penalty=self.length_penalty,
322
            include_stop_str_in_output=self.include_stop_str_in_output)
323

324
    def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
325
326
        # TODO(#9845): remove max_tokens when field is removed from OpenAI API
        max_tokens = self.max_completion_tokens or self.max_tokens
327
328
        if max_tokens is None:
            max_tokens = default_max_tokens
329

330
331
332
333
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

334
        guided_json_object = None
335
336
337
338
339
340
341
342
343
        if self.response_format is not None:
            if self.response_format.type == "json_object":
                guided_json_object = True
            elif self.response_format.type == "json_schema":
                json_schema = self.response_format.json_schema
                assert json_schema is not None
                self.guided_json = json_schema.json_schema
                if self.guided_decoding_backend is None:
                    self.guided_decoding_backend = "lm-format-enforcer"
344
345
346
347
348
349
350
351
352

        guided_decoding = GuidedDecodingParams.from_optional(
            json=self._get_guided_json_from_tool() or self.guided_json,
            regex=self.guided_regex,
            choice=self.guided_choice,
            grammar=self.guided_grammar,
            json_object=guided_json_object,
            backend=self.guided_decoding_backend,
            whitespace_pattern=self.guided_whitespace_pattern)
353

354
        return SamplingParams.from_optional(
355
            n=self.n,
356
            best_of=self.best_of,
357
358
359
360
361
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
362
            top_k=self.top_k,
363
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
364
            seed=self.seed,
365
366
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
367
            logprobs=self.top_logprobs if self.logprobs else None,
368
            prompt_logprobs=prompt_logprobs,
369
            ignore_eos=self.ignore_eos,
370
            max_tokens=max_tokens,
371
            min_tokens=self.min_tokens,
372
373
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
374
            include_stop_str_in_output=self.include_stop_str_in_output,
375
            truncate_prompt_tokens=self.truncate_prompt_tokens,
376
377
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
            guided_decoding=guided_decoding,
            logit_bias=self.logit_bias)

    def _get_guided_json_from_tool(
            self) -> Optional[Union[str, dict, BaseModel]]:
        # user has chosen to not use any tool
        if self.tool_choice == "none" or self.tools is None:
            return None

        # user has chosen to use a named tool
        if type(self.tool_choice) is ChatCompletionNamedToolChoiceParam:
            tool_name = self.tool_choice.function.name
            tools = {tool.function.name: tool.function for tool in self.tools}
            if tool_name not in tools:
                raise ValueError(
                    f"Tool '{tool_name}' has not been passed in `tools`.")
            tool = tools[tool_name]
            return tool.parameters

        return None
398

399
    @model_validator(mode="before")
400
    @classmethod
401
402
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
403
            raise ValueError(
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
                "Stream options can only be defined when `stream=True`.")

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (top_logprobs := data.get("top_logprobs")) is not None:
            if top_logprobs < 0:
                raise ValueError("`top_logprobs` must be a positive value.")

            if not data.get("logprobs"):
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )

        return data
429

430
431
432
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
433
434
435
        if isinstance(data, ValueError):
            raise data

436
437
438
439
440
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
441
        # you can only use one kind of guided decoding
442
443
444
445
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
446
        # you can only either use guided decoding or tools, not both
447
448
        if guide_count > 1 and data.get("tool_choice",
                                        "none") not in ("none", "auto"):
449
450
451
452
453
454
            raise ValueError(
                "You can only either use guided decoding or tools, not both.")
        return data

    @model_validator(mode="before")
    @classmethod
455
456
457
458
    def check_tool_usage(cls, data):

        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
459
        if "tool_choice" not in data and data.get("tools"):
460
461
            data["tool_choice"] = "auto"

462
463
464
465
466
467
        # if "tool_choice" is "none" -- ignore tools if present
        if "tool_choice" in data and data["tool_choice"] == "none":
            # ensure that no tools are present
            data.pop("tools", None)
            return data

468
469
470
471
        # if "tool_choice" is specified -- validation
        if "tool_choice" in data:

            # ensure that if "tool choice" is specified, tools are present
472
473
474
            if "tools" not in data or data["tools"] is None:
                raise ValueError(
                    "When using `tool_choice`, `tools` must be set.")
475
476
477
478
479
480

            # make sure that tool choice is either a named tool
            # OR that it's set to "auto"
            if data["tool_choice"] != "auto" and not isinstance(
                    data["tool_choice"], dict):
                raise ValueError(
481
482
                    "`tool_choice` must either be a named tool, \"auto\", "
                    "or \"none\".")
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507

            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
                specified_function = data["tool_choice"]["function"]
                if not specified_function:
                    raise ValueError(
                        "Incorrectly formatted `tool_choice`. Should be like "
                        "`{\"type\": \"function\","
                        " \"function\": {\"name\": \"my_function\"}}`")
                specified_function_name = specified_function["name"]
                if not specified_function_name:
                    raise ValueError(
                        "Incorrectly formatted `tool_choice`. Should be like "
                        "`{\"type\": \"function\", "
                        "\"function\": {\"name\": \"my_function\"}}`")
                for tool in data["tools"]:
                    if tool["function"]["name"] == specified_function_name:
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
                        " of the specified `tools`")
508
509
        return data

510
511
512
513
514
515
516
517
518
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

Zhuohan Li's avatar
Zhuohan Li committed
519

520
class CompletionRequest(OpenAIBaseModel):
521
522
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
Zhuohan Li's avatar
Zhuohan Li committed
523
    model: str
524
    prompt: Union[List[int], List[List[int]], str, List[str]]
525
    best_of: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
526
527
528
    echo: Optional[bool] = False
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
529
530
    logprobs: Optional[int] = None
    max_tokens: Optional[int] = 16
531
    n: int = 1
532
    presence_penalty: Optional[float] = 0.0
533
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
534
535
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
    stream: Optional[bool] = False
536
    stream_options: Optional[StreamOptions] = None
537
538
539
    suffix: Optional[str] = None
    temperature: Optional[float] = 1.0
    top_p: Optional[float] = 1.0
Zhuohan Li's avatar
Zhuohan Li committed
540
    user: Optional[str] = None
541
542

    # doc: begin-completion-sampling-params
543
544
545
546
547
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
548
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
549
550
551
552
553
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
554
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
555
    allowed_token_ids: Optional[List[int]] = None
556
    prompt_logprobs: Optional[int] = None
557
558
559
    # doc: end-completion-sampling-params

    # doc: begin-completion-extra-params
560
561
    add_special_tokens: bool = Field(
        default=True,
562
        description=(
563
564
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."),
565
566
567
568
569
    )
    response_format: Optional[ResponseFormat] = Field(
        default=None,
        description=
        ("Similar to chat completion, this parameter specifies the format of "
570
571
         "output. Only {'type': 'json_object'}, {'type': 'json_schema'} or "
         "{'type': 'text' } is supported."),
572
573
574
    )
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
575
        description="If specified, the output will follow the JSON schema.",
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
592
593
594
595
596
597
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be one of "
            "'outlines' / 'lm-format-enforcer'"))
598
599
600
601
602
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
603
604
605
606
607
608
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
609
610

    # doc: end-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
611

612
613
614
615
616
617
618
619
620
621
622
623
624
625
    def to_beam_search_params(self,
                              default_max_tokens: int) -> BeamSearchParams:
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens

        n = self.n if self.n is not None else 1
        temperature = self.temperature if self.temperature is not None else 0.0

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
626
            length_penalty=self.length_penalty,
627
            include_stop_str_in_output=self.include_stop_str_in_output)
628

629
    def to_sampling_params(self, default_max_tokens: int) -> SamplingParams:
630
631
632
633
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens

634
635
636
637
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

638
639
        echo_without_generation = self.echo and self.max_tokens == 0

640
641
642
643
644
645
646
647
648
649
650
651
652
        guided_json_object = None
        if (self.response_format is not None
                and self.response_format.type == "json_object"):
            guided_json_object = True

        guided_decoding = GuidedDecodingParams.from_optional(
            json=self.guided_json,
            regex=self.guided_regex,
            choice=self.guided_choice,
            grammar=self.guided_grammar,
            json_object=guided_json_object,
            backend=self.guided_decoding_backend,
            whitespace_pattern=self.guided_whitespace_pattern)
653

654
        return SamplingParams.from_optional(
655
656
657
658
659
660
661
662
663
            n=self.n,
            best_of=self.best_of,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
            top_k=self.top_k,
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
664
            seed=self.seed,
665
666
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
667
            logprobs=self.logprobs,
668
            ignore_eos=self.ignore_eos,
669
            max_tokens=max_tokens if not echo_without_generation else 1,
670
            min_tokens=self.min_tokens,
671
            prompt_logprobs=prompt_logprobs,
672
            skip_special_tokens=self.skip_special_tokens,
673
            spaces_between_special_tokens=self.spaces_between_special_tokens,
674
            include_stop_str_in_output=self.include_stop_str_in_output,
675
            truncate_prompt_tokens=self.truncate_prompt_tokens,
676
677
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
678
679
680
            guided_decoding=guided_decoding,
            logit_bias=self.logit_bias,
            allowed_token_ids=self.allowed_token_ids)
681

682
683
684
685
686
687
688
689
690
691
692
693
694
695
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
        return data

696
697
698
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
699
700
701
702
703
704
705
706
707
708
709
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
            raise ValueError("`logprobs` must be a positive value.")

710
711
        return data

712
713
714
715
716
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
            raise ValueError(
717
718
                "Stream options can only be defined when `stream=True`.")

719
720
        return data

Zhuohan Li's avatar
Zhuohan Li committed
721

722
class EmbeddingCompletionRequest(OpenAIBaseModel):
723
724
725
726
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
    model: str
    input: Union[List[int], List[List[int]], str, List[str]]
727
    encoding_format: Literal["float", "base64"] = "float"
728
729
    dimensions: Optional[int] = None
    user: Optional[str] = None
730
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
731
732
733
734
735

    # doc: begin-embedding-pooling-params
    additional_data: Optional[Any] = None
    # doc: end-embedding-pooling-params

736
    # doc: begin-embedding-extra-params
737
738
739
740
741
742
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."),
    )
743
744
745
746
747
748
749
750
751
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))

    # doc: end-embedding-extra-params

752
753
754
755
    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
class EmbeddingChatRequest(OpenAIBaseModel):
    model: str
    messages: List[ChatCompletionMessageParam]

    encoding_format: Literal["float", "base64"] = "float"
    dimensions: Optional[int] = None
    user: Optional[str] = None
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

    # doc: begin-chat-embedding-pooling-params
    additional_data: Optional[Any] = None
    # doc: end-chat-embedding-pooling-params

    # doc: begin-chat-embedding-extra-params
    add_generation_prompt: bool = Field(
        default=True,
        description=
        ("If true, the generation prompt will be added to the chat template. "
         "This is a parameter used by chat template in tokenizer config of the "
         "model."),
    )
    continue_final_message: bool = Field(
        default=False,
        description=
        ("If this is set, the chat will be formatted so that the final "
         "message in the chat is open-ended, without any EOS tokens. The "
         "model will continue this message rather than starting a new one. "
         "This allows you to \"prefill\" part of the model's response for it. "
         "Cannot be used at the same time as `add_generation_prompt`."),
    )
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
            "default)."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."),
    )
    chat_template_kwargs: Optional[Dict[str, Any]] = Field(
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
    # doc: end-chat-embedding-extra-params

    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


EmbeddingRequest = Union[EmbeddingCompletionRequest, EmbeddingChatRequest]


832
class CompletionLogProbs(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
833
834
835
    text_offset: List[int] = Field(default_factory=list)
    token_logprobs: List[Optional[float]] = Field(default_factory=list)
    tokens: List[str] = Field(default_factory=list)
836
837
    top_logprobs: List[Optional[Dict[str,
                                     float]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
838
839


840
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
841
842
    index: int
    text: str
843
    logprobs: Optional[CompletionLogProbs] = None
844
845
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
846
847
848
849
850
851
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
852
    prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
Zhuohan Li's avatar
Zhuohan Li committed
853
854


855
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
856
857
858
859
860
861
862
863
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseChoice]
    usage: UsageInfo


864
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
865
866
    index: int
    text: str
867
    logprobs: Optional[CompletionLogProbs] = None
868
869
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
870
871
872
873
874
875
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
Zhuohan Li's avatar
Zhuohan Li committed
876
877


878
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
879
880
881
882
883
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseStreamChoice]
884
    usage: Optional[UsageInfo] = Field(default=None)
885
886


887
class EmbeddingResponseData(OpenAIBaseModel):
888
889
    index: int
    object: str = "embedding"
890
    embedding: Union[List[float], str]
891
892


893
class EmbeddingResponse(OpenAIBaseModel):
894
    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
895
896
897
898
899
900
901
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    data: List[EmbeddingResponseData]
    usage: UsageInfo


902
903
904
905
906
907
908
909
910
911
912
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    function: FunctionCall


913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
class DeltaFunctionCall(BaseModel):
    name: Optional[str] = None
    arguments: Optional[str] = None


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    index: int
    function: Optional[DeltaFunctionCall] = None


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
    tool_calls: List[ToolCall]

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
    content: Optional[str] = None


938
class ChatMessage(OpenAIBaseModel):
939
    role: str
940
    content: Optional[str] = None
941
    tool_calls: List[ToolCall] = Field(default_factory=list)
942
943


944
945
946
947
948
949
950
951
952
953
954
955
956
957
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
    bytes: Optional[List[int]] = None


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
    top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list)


class ChatCompletionLogProbs(OpenAIBaseModel):
    content: Optional[List[ChatCompletionLogProbsContent]] = None


958
class ChatCompletionResponseChoice(OpenAIBaseModel):
959
960
    index: int
    message: ChatMessage
961
    logprobs: Optional[ChatCompletionLogProbs] = None
962
963
964
    # per OpenAI spec this is the default
    finish_reason: Optional[str] = "stop"
    # not part of the OpenAI spec but included in vLLM for legacy reasons
965
    stop_reason: Optional[Union[int, str]] = None
966
967


968
class ChatCompletionResponse(OpenAIBaseModel):
969
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
970
    object: Literal["chat.completion"] = "chat.completion"
971
972
973
974
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseChoice]
    usage: UsageInfo
975
    prompt_logprobs: Optional[List[Optional[Dict[int, Logprob]]]] = None
976
977


978
class DeltaMessage(OpenAIBaseModel):
979
980
    role: Optional[str] = None
    content: Optional[str] = None
981
    tool_calls: List[DeltaToolCall] = Field(default_factory=list)
982
983


984
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
985
986
    index: int
    delta: DeltaMessage
987
    logprobs: Optional[ChatCompletionLogProbs] = None
988
    finish_reason: Optional[str] = None
989
    stop_reason: Optional[Union[int, str]] = None
990
991


992
class ChatCompletionStreamResponse(OpenAIBaseModel):
993
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
994
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
995
996
997
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseStreamChoice]
998
    usage: Optional[UsageInfo] = Field(default=None)
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019


class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

1020
    # The parameters of the request.
1021
    body: Union[ChatCompletionRequest, EmbeddingRequest]
1022
1023


1024
1025
1026
1027
1028
1029
1030
1031
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
1032
    body: Optional[Union[ChatCompletionResponse, EmbeddingResponse]] = None
1033
1034


1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

1046
    response: Optional[BatchResponseData]
1047
1048
1049
1050

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
    error: Optional[Any]
1051
1052


1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
class TokenizeCompletionRequest(OpenAIBaseModel):
    model: str
    prompt: str

    add_special_tokens: bool = Field(default=True)


class TokenizeChatRequest(OpenAIBaseModel):
    model: str
    messages: List[ChatCompletionMessageParam]

1064
    add_generation_prompt: bool = Field(default=True)
1065
    continue_final_message: bool = Field(default=False)
1066
    add_special_tokens: bool = Field(default=False)
1067

1068
1069
1070
1071
1072
1073
1074
1075
1076
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

1077
1078

TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
1079
1080
1081
1082
1083


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
1084
    tokens: List[int]
1085
1086
1087
1088
1089
1090
1091
1092
1093


class DetokenizeRequest(OpenAIBaseModel):
    model: str
    tokens: List[int]


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103


class LoadLoraAdapterRequest(BaseModel):
    lora_name: str
    lora_path: str


class UnloadLoraAdapterRequest(BaseModel):
    lora_name: str
    lora_int_id: Optional[int] = Field(default=None)