protocol.py 59.5 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
4
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
5
import re
Zhuohan Li's avatar
Zhuohan Li committed
6
import time
7
from argparse import Namespace
8
from typing import Annotated, Any, ClassVar, Literal, Optional, Union
Zhuohan Li's avatar
Zhuohan Li committed
9

10
import torch
11
from fastapi import UploadFile
12
13
from pydantic import (BaseModel, ConfigDict, Field, TypeAdapter,
                      ValidationInfo, field_validator, model_validator)
14
from typing_extensions import TypeAlias
Zhuohan Li's avatar
Zhuohan Li committed
15

16
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
17
from vllm.logger import init_logger
18
from vllm.pooling_params import PoolingParams
19
20
from vllm.sampling_params import (BeamSearchParams, GuidedDecodingParams,
                                  RequestOutputKind, SamplingParams)
21
from vllm.sequence import Logprob
22
from vllm.utils import random_uuid, resolve_obj_by_qualname
23

24
25
logger = init_logger(__name__)

26
27
28
# torch is mocked during docs generation,
# so we have to provide the values as literals
_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
29
_LONG_INFO: Union["torch.iinfo", Namespace]
30
31
32
33
34
35
36
37
38
39
40
41
42
43

try:
    from sphinx.ext.autodoc.mock import _MockModule

    if isinstance(torch, _MockModule):
        _LONG_INFO = _MOCK_LONG_INFO
    else:
        _LONG_INFO = torch.iinfo(torch.long)
except ModuleNotFoundError:
    _LONG_INFO = torch.iinfo(torch.long)

assert _LONG_INFO.min == _MOCK_LONG_INFO.min
assert _LONG_INFO.max == _MOCK_LONG_INFO.max

Zhuohan Li's avatar
Zhuohan Li committed
44

45
class OpenAIBaseModel(BaseModel):
46
47
48
    # OpenAI API does allow extra fields
    model_config = ConfigDict(extra="allow")

49
    # Cache class field names
50
    field_names: ClassVar[Optional[set[str]]] = None
51

52
    @model_validator(mode="wrap")
53
    @classmethod
54
55
56
57
    def __log_extra_fields__(cls, data, handler):
        result = handler(data)
        if not isinstance(data, dict):
            return result
58
59
        field_names = cls.field_names
        if field_names is None:
60
61
62
63
            # Get all class field names and their potential aliases
            field_names = set()
            for field_name, field in cls.model_fields.items():
                field_names.add(field_name)
64
65
66
67
68
69
70
71
72
73
                if alias := getattr(field, 'alias', None):
                    field_names.add(alias)
            cls.field_names = field_names

        # Compare against both field names and aliases
        if any(k not in field_names for k in data):
            logger.warning(
                "The following fields were present in the request "
                "but ignored: %s",
                data.keys() - field_names)
74
        return result
75
76
77


class ErrorResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
78
79
80
81
    object: str = "error"
    message: str
    type: str
    param: Optional[str] = None
82
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
83
84


85
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
86
87
88
89
90
91
92
93
94
95
96
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
    group: Optional[str] = None
97
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
98
99


100
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
101
102
103
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
104
    owned_by: str = "vllm"
Zhuohan Li's avatar
Zhuohan Li committed
105
106
    root: Optional[str] = None
    parent: Optional[str] = None
107
    max_model_len: Optional[int] = None
108
    permission: list[ModelPermission] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
109
110


111
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
112
    object: str = "list"
113
    data: list[ModelCard] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
114
115


116
117
118
119
class PromptTokenUsageInfo(OpenAIBaseModel):
    cached_tokens: Optional[int] = None


120
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
121
122
123
    prompt_tokens: int = 0
    total_tokens: int = 0
    completion_tokens: Optional[int] = 0
124
    prompt_tokens_details: Optional[PromptTokenUsageInfo] = None
Zhuohan Li's avatar
Zhuohan Li committed
125
126


127
128
129
130
131
class RequestResponseMetadata(BaseModel):
    request_id: str
    final_usage_info: Optional[UsageInfo] = None


132
133
134
135
136
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
137
    json_schema: Optional[dict[str, Any]] = Field(default=None, alias='schema')
138
139
140
    strict: Optional[bool] = None


141
class ResponseFormat(OpenAIBaseModel):
142
143
144
    # type must be "json_schema", "json_object" or "text"
    type: Literal["text", "json_object", "json_schema"]
    json_schema: Optional[JsonSchemaResponseFormat] = None
145
146


147
class StreamOptions(OpenAIBaseModel):
148
    include_usage: Optional[bool] = True
149
    continuous_usage_stats: Optional[bool] = False
150
151


152
153
154
class FunctionDefinition(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
155
    parameters: Optional[dict[str, Any]] = None
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


172
173
class LogitsProcessorConstructor(BaseModel):
    qualname: str
174
175
    args: Optional[list[Any]] = None
    kwargs: Optional[dict[str, Any]] = None
176
177


178
LogitsProcessors = list[Union[str, LogitsProcessorConstructor]]
179
180
181


def get_logits_processors(processors: Optional[LogitsProcessors],
182
                          pattern: Optional[str]) -> Optional[list[Any]]:
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
    if processors and pattern:
        logits_processors = []
        for processor in processors:
            qualname = processor if isinstance(processor,
                                               str) else processor.qualname
            if not re.match(pattern, qualname):
                raise ValueError(
                    f"Logits processor '{qualname}' is not allowed by this "
                    "server. See --logits-processor-pattern engine argument "
                    "for more information.")
            try:
                logits_processor = resolve_obj_by_qualname(qualname)
            except Exception as e:
                raise ValueError(
                    f"Logits processor '{qualname}' could not be resolved: {e}"
                ) from e
            if isinstance(processor, LogitsProcessorConstructor):
                logits_processor = logits_processor(*processor.args or [],
                                                    **processor.kwargs or {})
            logits_processors.append(logits_processor)
        return logits_processors
    elif processors:
        raise ValueError(
            "The `logits_processors` argument is not supported by this "
            "server. See --logits-processor-pattern engine argugment "
            "for more information.")
    return None


212
class ChatCompletionRequest(OpenAIBaseModel):
213
214
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
215
    messages: list[ChatCompletionMessageParam]
216
    model: Optional[str] = None
217
    frequency_penalty: Optional[float] = 0.0
218
    logit_bias: Optional[dict[str, float]] = None
219
    logprobs: Optional[bool] = False
220
    top_logprobs: Optional[int] = 0
221
222
223
224
225
226
    # TODO(#9845): remove max_tokens when field is removed from OpenAI API
    max_tokens: Optional[int] = Field(
        default=None,
        deprecated=
        'max_tokens is deprecated in favor of the max_completion_tokens field')
    max_completion_tokens: Optional[int] = None
227
228
229
    n: Optional[int] = 1
    presence_penalty: Optional[float] = 0.0
    response_format: Optional[ResponseFormat] = None
230
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
231
    stop: Optional[Union[str, list[str]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
232
    stream: Optional[bool] = False
233
    stream_options: Optional[StreamOptions] = None
234
235
    temperature: Optional[float] = None
    top_p: Optional[float] = None
236
    tools: Optional[list[ChatCompletionToolsParam]] = None
237
    tool_choice: Optional[Union[Literal["none"], Literal["auto"],
238
                                ChatCompletionNamedToolChoiceParam]] = "none"
239
240
241

    # NOTE this will be ignored by VLLM -- the model determines the behavior
    parallel_tool_calls: Optional[bool] = False
Zhuohan Li's avatar
Zhuohan Li committed
242
    user: Optional[str] = None
243
244

    # doc: begin-chat-completion-sampling-params
245
    best_of: Optional[int] = None
246
    use_beam_search: bool = False
247
248
249
    top_k: Optional[int] = None
    min_p: Optional[float] = None
    repetition_penalty: Optional[float] = None
250
    length_penalty: float = 1.0
251
    stop_token_ids: Optional[list[int]] = Field(default_factory=list)
252
253
254
255
256
257
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
258
    prompt_logprobs: Optional[int] = None
259
260
261
    # doc: end-chat-completion-sampling-params

    # doc: begin-chat-completion-extra-params
262
    echo: bool = Field(
263
264
265
266
267
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
            "if they belong to the same role."),
    )
268
    add_generation_prompt: bool = Field(
269
270
271
272
273
274
        default=True,
        description=
        ("If true, the generation prompt will be added to the chat template. "
         "This is a parameter used by chat template in tokenizer config of the "
         "model."),
    )
275
276
277
278
279
280
281
282
283
    continue_final_message: bool = Field(
        default=False,
        description=
        ("If this is set, the chat will be formatted so that the final "
         "message in the chat is open-ended, without any EOS tokens. The "
         "model will continue this message rather than starting a new one. "
         "This allows you to \"prefill\" part of the model's response for it. "
         "Cannot be used at the same time as `add_generation_prompt`."),
    )
284
    add_special_tokens: bool = Field(
285
286
287
288
289
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
290
            "special tokens so this should be set to false (as is the "
291
292
            "default)."),
    )
293
    documents: Optional[list[dict[str, str]]] = Field(
294
295
296
297
298
299
300
301
302
303
304
305
        default=None,
        description=
        ("A list of dicts representing documents that will be accessible to "
         "the model if it is performing RAG (retrieval-augmented generation)."
         " If the template does not support RAG, this argument will have no "
         "effect. We recommend that each document should be a dict containing "
         "\"title\" and \"text\" keys."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
306
307
308
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."),
309
    )
310
    chat_template_kwargs: Optional[dict[str, Any]] = Field(
311
312
313
314
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
315
    mm_processor_kwargs: Optional[dict[str, Any]] = Field(
316
317
318
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
319
320
321
322
323
324
325
326
327
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
        description=("If specified, the output will follow the JSON schema."),
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
328
    guided_choice: Optional[list[str]] = Field(
329
330
331
332
333
334
335
336
337
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
338
339
340
341
342
343
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be either "
            "'outlines' / 'lm-format-enforcer'"))
344
345
346
347
348
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
349
350
351
352
353
354
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
355
356
357
358
359
360
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."))
361
362
363
364
365
366
367
368
369
370
371
    logits_processors: Optional[LogitsProcessors] = Field(
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
            "{'param': 'value'}}."))
372
373
374
375
376
377
    return_tokens_as_token_ids: Optional[bool] = Field(
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
            "that are not JSON-encodable can be identified."))
378
379

    # doc: end-chat-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
380

381
382
383
384
385
386
387
388
389
390
391
392
393
394
    # Default sampling parameters for chat completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
        "top_k": -1,
        "min_p": 0.0,
    }

    def to_beam_search_params(
            self,
            default_max_tokens: int,
            default_sampling_params: Optional[dict] = None
    ) -> BeamSearchParams:
395
396
        # TODO(#9845): remove max_tokens when field is removed from OpenAI API
        max_tokens = self.max_completion_tokens or self.max_tokens
397

398
399
        if default_sampling_params is None:
            default_sampling_params = {}
400
        n = self.n if self.n is not None else 1
401

402
403
404
405
406
407
        # Use minimum of context window, user request & server limit.
        max_tokens = min(
            val for val in (default_max_tokens, max_tokens,
                            default_sampling_params.get("max_tokens", None))
            if val is not None)

408
409
410
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
411
412
413
414
415
416

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
417
            length_penalty=self.length_penalty,
418
            include_stop_str_in_output=self.include_stop_str_in_output)
419

420
    def to_sampling_params(
421
422
423
424
            self,
            default_max_tokens: int,
            logits_processor_pattern: Optional[str],
            default_sampling_params: Optional[dict] = None) -> SamplingParams:
425
426
        # TODO(#9845): remove max_tokens when field is removed from OpenAI API
        max_tokens = self.max_completion_tokens or self.max_tokens
427

428
429
        if default_sampling_params is None:
            default_sampling_params = {}
430
431
432
433
434
435
436

        # Use minimum of context window, user request & server limit.
        max_tokens = min(
            val for val in (default_max_tokens, max_tokens,
                            default_sampling_params.get("max_tokens", None))
            if val is not None)

437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"])
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"])
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"])

456
457
458
459
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

460
        guided_json_object = None
461
462
463
464
465
466
467
468
        if self.response_format is not None:
            if self.response_format.type == "json_object":
                guided_json_object = True
            elif self.response_format.type == "json_schema":
                json_schema = self.response_format.json_schema
                assert json_schema is not None
                self.guided_json = json_schema.json_schema
                if self.guided_decoding_backend is None:
469
                    self.guided_decoding_backend = "xgrammar"
470
471
472
473
474
475
476
477
478

        guided_decoding = GuidedDecodingParams.from_optional(
            json=self._get_guided_json_from_tool() or self.guided_json,
            regex=self.guided_regex,
            choice=self.guided_choice,
            grammar=self.guided_grammar,
            json_object=guided_json_object,
            backend=self.guided_decoding_backend,
            whitespace_pattern=self.guided_whitespace_pattern)
479

480
        return SamplingParams.from_optional(
481
            n=self.n,
482
            best_of=self.best_of,
483
484
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
485
486
487
488
489
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
490
            seed=self.seed,
491
492
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
493
            logprobs=self.top_logprobs if self.logprobs else None,
494
            prompt_logprobs=prompt_logprobs,
495
            ignore_eos=self.ignore_eos,
496
            max_tokens=max_tokens,
497
            min_tokens=self.min_tokens,
498
499
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
500
501
            logits_processors=get_logits_processors(self.logits_processors,
                                                    logits_processor_pattern),
502
            include_stop_str_in_output=self.include_stop_str_in_output,
503
            truncate_prompt_tokens=self.truncate_prompt_tokens,
504
505
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
            guided_decoding=guided_decoding,
            logit_bias=self.logit_bias)

    def _get_guided_json_from_tool(
            self) -> Optional[Union[str, dict, BaseModel]]:
        # user has chosen to not use any tool
        if self.tool_choice == "none" or self.tools is None:
            return None

        # user has chosen to use a named tool
        if type(self.tool_choice) is ChatCompletionNamedToolChoiceParam:
            tool_name = self.tool_choice.function.name
            tools = {tool.function.name: tool.function for tool in self.tools}
            if tool_name not in tools:
                raise ValueError(
                    f"Tool '{tool_name}' has not been passed in `tools`.")
            tool = tools[tool_name]
            return tool.parameters

        return None
526

527
    @model_validator(mode="before")
528
    @classmethod
529
530
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
531
            raise ValueError(
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
                "Stream options can only be defined when `stream=True`.")

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (top_logprobs := data.get("top_logprobs")) is not None:
            if top_logprobs < 0:
                raise ValueError("`top_logprobs` must be a positive value.")

            if not data.get("logprobs"):
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )

        return data
557

558
559
560
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
561
562
563
        if isinstance(data, ValueError):
            raise data

564
565
566
567
568
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
569
        # you can only use one kind of guided decoding
570
571
572
573
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
574
        # you can only either use guided decoding or tools, not both
575
576
        if guide_count > 1 and data.get("tool_choice",
                                        "none") not in ("none", "auto"):
577
578
579
580
581
582
            raise ValueError(
                "You can only either use guided decoding or tools, not both.")
        return data

    @model_validator(mode="before")
    @classmethod
583
584
585
586
    def check_tool_usage(cls, data):

        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
587
        if "tool_choice" not in data and data.get("tools"):
588
589
            data["tool_choice"] = "auto"

590
591
592
593
594
595
        # if "tool_choice" is "none" -- ignore tools if present
        if "tool_choice" in data and data["tool_choice"] == "none":
            # ensure that no tools are present
            data.pop("tools", None)
            return data

596
597
598
599
        # if "tool_choice" is specified -- validation
        if "tool_choice" in data:

            # ensure that if "tool choice" is specified, tools are present
600
601
602
            if "tools" not in data or data["tools"] is None:
                raise ValueError(
                    "When using `tool_choice`, `tools` must be set.")
603
604
605
606
607
608

            # make sure that tool choice is either a named tool
            # OR that it's set to "auto"
            if data["tool_choice"] != "auto" and not isinstance(
                    data["tool_choice"], dict):
                raise ValueError(
609
610
                    "`tool_choice` must either be a named tool, \"auto\", "
                    "or \"none\".")
611
612
613
614
615

            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
616
                specified_function = data["tool_choice"].get("function")
617
618
                if not specified_function:
                    raise ValueError(
619
620
                        "Expected field `function` in `tool_choice`."
                        " Correct usage: `{\"type\": \"function\","
621
                        " \"function\": {\"name\": \"my_function\"}}`")
622
                specified_function_name = specified_function.get("name")
623
624
                if not specified_function_name:
                    raise ValueError(
625
626
                        "Expected field `name` in `function` in `tool_choice`."
                        "Correct usage: `{\"type\": \"function\", "
627
628
629
630
631
632
633
634
635
                        "\"function\": {\"name\": \"my_function\"}}`")
                for tool in data["tools"]:
                    if tool["function"]["name"] == specified_function_name:
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
                        " of the specified `tools`")
636
637
        return data

638
639
640
641
642
643
644
645
646
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

Zhuohan Li's avatar
Zhuohan Li committed
647

648
class CompletionRequest(OpenAIBaseModel):
649
650
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
651
    model: Optional[str] = None
652
    prompt: Union[list[int], list[list[int]], str, list[str]]
653
    best_of: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
654
655
    echo: Optional[bool] = False
    frequency_penalty: Optional[float] = 0.0
656
    logit_bias: Optional[dict[str, float]] = None
657
658
    logprobs: Optional[int] = None
    max_tokens: Optional[int] = 16
659
    n: int = 1
660
    presence_penalty: Optional[float] = 0.0
661
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
662
    stop: Optional[Union[str, list[str]]] = Field(default_factory=list)
663
    stream: Optional[bool] = False
664
    stream_options: Optional[StreamOptions] = None
665
    suffix: Optional[str] = None
666
667
    temperature: Optional[float] = None
    top_p: Optional[float] = None
Zhuohan Li's avatar
Zhuohan Li committed
668
    user: Optional[str] = None
669
670

    # doc: begin-completion-sampling-params
671
    use_beam_search: bool = False
672
673
674
    top_k: Optional[int] = None
    min_p: Optional[float] = None
    repetition_penalty: Optional[float] = None
675
    length_penalty: float = 1.0
676
    stop_token_ids: Optional[list[int]] = Field(default_factory=list)
677
678
679
680
681
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
682
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
683
    allowed_token_ids: Optional[list[int]] = None
684
    prompt_logprobs: Optional[int] = None
685
686
687
    # doc: end-completion-sampling-params

    # doc: begin-completion-extra-params
688
689
    add_special_tokens: bool = Field(
        default=True,
690
        description=(
691
692
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."),
693
694
695
696
697
    )
    response_format: Optional[ResponseFormat] = Field(
        default=None,
        description=
        ("Similar to chat completion, this parameter specifies the format of "
698
699
         "output. Only {'type': 'json_object'}, {'type': 'json_schema'} or "
         "{'type': 'text' } is supported."),
700
701
702
    )
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
703
        description="If specified, the output will follow the JSON schema.",
704
705
706
707
708
709
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
710
    guided_choice: Optional[list[str]] = Field(
711
712
713
714
715
716
717
718
719
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
720
721
722
723
724
725
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be one of "
            "'outlines' / 'lm-format-enforcer'"))
726
727
728
729
730
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
731
732
733
734
735
736
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
737
738
739
740
741
742
743
744
745
746
747
    logits_processors: Optional[LogitsProcessors] = Field(
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
            "{'param': 'value'}}."))
748
749
750
751
752
753
    return_tokens_as_token_ids: Optional[bool] = Field(
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
            "that are not JSON-encodable can be identified."))
754
755

    # doc: end-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
756

757
758
759
760
761
762
763
764
765
766
767
768
769
770
    # Default sampling parameters for completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
        "top_k": -1,
        "min_p": 0.0,
    }

    def to_beam_search_params(
            self,
            default_max_tokens: int,
            default_sampling_params: Optional[dict] = None
    ) -> BeamSearchParams:
771
772
        max_tokens = self.max_tokens

773
774
        if default_sampling_params is None:
            default_sampling_params = {}
775
        n = self.n if self.n is not None else 1
776

777
778
779
780
781
782
        # Use minimum of context window, user request & server limit.
        max_tokens = min(
            val for val in (default_max_tokens, max_tokens,
                            default_sampling_params.get("max_tokens", None))
            if val is not None)

783
784
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get("temperature", 1.0)
785
786
787
788
789
790

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
791
            length_penalty=self.length_penalty,
792
            include_stop_str_in_output=self.include_stop_str_in_output)
793

794
    def to_sampling_params(
795
796
797
798
            self,
            default_max_tokens: int,
            logits_processor_pattern: Optional[str],
            default_sampling_params: Optional[dict] = None) -> SamplingParams:
799
800
        max_tokens = self.max_tokens

801
802
        if default_sampling_params is None:
            default_sampling_params = {}
803
804
805
806
807
808
809

        # Use minimum of context window, user request & server limit.
        max_tokens = min(
            val for val in (default_max_tokens, max_tokens,
                            default_sampling_params.get("max_tokens", None))
            if val is not None)

810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"])
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"])
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"])

829
830
831
832
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

833
834
        echo_without_generation = self.echo and self.max_tokens == 0

835
836
837
838
839
840
841
842
843
844
845
846
847
        guided_json_object = None
        if (self.response_format is not None
                and self.response_format.type == "json_object"):
            guided_json_object = True

        guided_decoding = GuidedDecodingParams.from_optional(
            json=self.guided_json,
            regex=self.guided_regex,
            choice=self.guided_choice,
            grammar=self.guided_grammar,
            json_object=guided_json_object,
            backend=self.guided_decoding_backend,
            whitespace_pattern=self.guided_whitespace_pattern)
848

849
        return SamplingParams.from_optional(
850
            n=self.n,
851
            best_of=self.best_of,
852
853
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
854
855
856
857
858
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
859
            seed=self.seed,
860
861
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
862
            logprobs=self.logprobs,
863
            ignore_eos=self.ignore_eos,
864
            max_tokens=max_tokens if not echo_without_generation else 1,
865
            min_tokens=self.min_tokens,
866
            prompt_logprobs=prompt_logprobs,
867
            skip_special_tokens=self.skip_special_tokens,
868
            spaces_between_special_tokens=self.spaces_between_special_tokens,
869
            include_stop_str_in_output=self.include_stop_str_in_output,
870
871
            logits_processors=get_logits_processors(self.logits_processors,
                                                    logits_processor_pattern),
872
            truncate_prompt_tokens=self.truncate_prompt_tokens,
873
874
            output_kind=RequestOutputKind.DELTA if self.stream \
                else RequestOutputKind.FINAL_ONLY,
875
876
877
            guided_decoding=guided_decoding,
            logit_bias=self.logit_bias,
            allowed_token_ids=self.allowed_token_ids)
878

879
880
881
882
883
884
885
886
887
888
889
890
891
892
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
        return data

893
894
895
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
896
897
898
899
900
901
902
903
904
905
906
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
            if data.get("stream") and prompt_logprobs > 0:
                raise ValueError(
                    "`prompt_logprobs` are not available when `stream=True`.")

            if prompt_logprobs < 0:
                raise ValueError("`prompt_logprobs` must be a positive value.")

        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
            raise ValueError("`logprobs` must be a positive value.")

907
908
        return data

909
910
911
912
913
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
            raise ValueError(
914
915
                "Stream options can only be defined when `stream=True`.")

916
917
        return data

Zhuohan Li's avatar
Zhuohan Li committed
918

919
class EmbeddingCompletionRequest(OpenAIBaseModel):
920
921
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
922
    model: Optional[str] = None
923
    input: Union[list[int], list[list[int]], str, list[str]]
924
    encoding_format: Literal["float", "base64"] = "float"
925
926
    dimensions: Optional[int] = None
    user: Optional[str] = None
927
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
928
929
930
931
932

    # doc: begin-embedding-pooling-params
    additional_data: Optional[Any] = None
    # doc: end-embedding-pooling-params

933
    # doc: begin-embedding-extra-params
934
935
936
937
938
939
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."),
    )
940
941
942
943
944
945
946
947
948
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))

    # doc: end-embedding-extra-params

949
950
951
952
    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


953
class EmbeddingChatRequest(OpenAIBaseModel):
954
    model: Optional[str] = None
955
    messages: list[ChatCompletionMessageParam]
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983

    encoding_format: Literal["float", "base64"] = "float"
    dimensions: Optional[int] = None
    user: Optional[str] = None
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

    # doc: begin-chat-embedding-pooling-params
    additional_data: Optional[Any] = None
    # doc: end-chat-embedding-pooling-params

    # doc: begin-chat-embedding-extra-params
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
            "default)."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."),
    )
984
    chat_template_kwargs: Optional[dict[str, Any]] = Field(
985
986
987
988
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
989
    mm_processor_kwargs: Optional[dict[str, Any]] = Field(
990
991
992
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))
    # doc: end-chat-embedding-extra-params

    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


EmbeddingRequest = Union[EmbeddingCompletionRequest, EmbeddingChatRequest]

1016
1017
1018
1019
PoolingCompletionRequest = EmbeddingCompletionRequest
PoolingChatRequest = EmbeddingChatRequest
PoolingRequest = Union[PoolingCompletionRequest, PoolingChatRequest]

1020

1021
class ScoreRequest(OpenAIBaseModel):
1022
    model: Optional[str] = None
1023
1024
    text_1: Union[list[str], str]
    text_2: Union[list[str], str]
1025
1026
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

1027
    # doc: begin-score-pooling-params
1028
    additional_data: Optional[Any] = None
1029
    # doc: end-score-pooling-params
1030

1031
    # doc: begin-score-extra-params
1032
1033
1034
1035
1036
1037
1038
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))

1039
1040
    # doc: end-score-extra-params

1041
1042
1043
1044
    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


1045
class RerankRequest(OpenAIBaseModel):
1046
    model: Optional[str] = None
1047
    query: str
1048
    documents: list[str]
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
    top_n: int = Field(default_factory=lambda: 0)
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None

    # doc: begin-rerank-pooling-params
    additional_data: Optional[Any] = None
    # doc: end-rerank-pooling-params

    # doc: begin-rerank-extra-params
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."))

    # doc: end-rerank-extra-params

    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


class RerankDocument(BaseModel):
    text: str


class RerankResult(BaseModel):
    index: int
    document: RerankDocument
    relevance_score: float


class RerankUsage(BaseModel):
    total_tokens: int


class RerankResponse(OpenAIBaseModel):
    id: str
    model: str
    usage: RerankUsage
1088
    results: list[RerankResult]
1089
1090


1091
class CompletionLogProbs(OpenAIBaseModel):
1092
1093
1094
1095
    text_offset: list[int] = Field(default_factory=list)
    token_logprobs: list[Optional[float]] = Field(default_factory=list)
    tokens: list[str] = Field(default_factory=list)
    top_logprobs: list[Optional[dict[str,
1096
                                     float]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
1097
1098


1099
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1100
1101
    index: int
    text: str
1102
    logprobs: Optional[CompletionLogProbs] = None
1103
1104
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
1105
1106
1107
1108
1109
1110
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
1111
    prompt_logprobs: Optional[list[Optional[dict[int, Logprob]]]] = None
Zhuohan Li's avatar
Zhuohan Li committed
1112
1113


1114
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1115
1116
1117
1118
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1119
    choices: list[CompletionResponseChoice]
Zhuohan Li's avatar
Zhuohan Li committed
1120
1121
1122
    usage: UsageInfo


1123
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1124
1125
    index: int
    text: str
1126
    logprobs: Optional[CompletionLogProbs] = None
1127
1128
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
1129
1130
1131
1132
1133
1134
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
Zhuohan Li's avatar
Zhuohan Li committed
1135
1136


1137
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1138
1139
1140
1141
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1142
    choices: list[CompletionResponseStreamChoice]
1143
    usage: Optional[UsageInfo] = Field(default=None)
1144
1145


1146
class EmbeddingResponseData(OpenAIBaseModel):
1147
1148
    index: int
    object: str = "embedding"
1149
    embedding: Union[list[float], str]
1150
1151


1152
class EmbeddingResponse(OpenAIBaseModel):
1153
    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
1154
1155
1156
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1157
    data: list[EmbeddingResponseData]
1158
1159
1160
    usage: UsageInfo


1161
1162
1163
class PoolingResponseData(OpenAIBaseModel):
    index: int
    object: str = "pooling"
1164
    data: Union[list[list[float]], list[float], str]
1165
1166
1167
1168
1169
1170
1171


class PoolingResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"pool-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1172
    data: list[PoolingResponseData]
1173
1174
1175
    usage: UsageInfo


1176
1177
1178
class ScoreResponseData(OpenAIBaseModel):
    index: int
    object: str = "score"
1179
    score: float
1180
1181
1182
1183
1184
1185
1186


class ScoreResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1187
    data: list[ScoreResponseData]
1188
1189
1190
    usage: UsageInfo


1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    function: FunctionCall


1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
class DeltaFunctionCall(BaseModel):
    name: Optional[str] = None
    arguments: Optional[str] = None


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    index: int
    function: Optional[DeltaFunctionCall] = None


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
1220
    tool_calls: list[ToolCall]
1221
1222
1223
1224
1225
1226

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
    content: Optional[str] = None


1227
class ChatMessage(OpenAIBaseModel):
1228
    role: str
1229
    reasoning_content: Optional[str] = None
1230
    content: Optional[str] = None
1231
    tool_calls: list[ToolCall] = Field(default_factory=list)
1232
1233


1234
1235
1236
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
1237
    bytes: Optional[list[int]] = None
1238
1239
1240


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
1241
    top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
1242
1243
1244


class ChatCompletionLogProbs(OpenAIBaseModel):
1245
    content: Optional[list[ChatCompletionLogProbsContent]] = None
1246
1247


1248
class ChatCompletionResponseChoice(OpenAIBaseModel):
1249
1250
    index: int
    message: ChatMessage
1251
    logprobs: Optional[ChatCompletionLogProbs] = None
1252
1253
1254
    # per OpenAI spec this is the default
    finish_reason: Optional[str] = "stop"
    # not part of the OpenAI spec but included in vLLM for legacy reasons
1255
    stop_reason: Optional[Union[int, str]] = None
1256
1257


1258
class ChatCompletionResponse(OpenAIBaseModel):
1259
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
1260
    object: Literal["chat.completion"] = "chat.completion"
1261
1262
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1263
    choices: list[ChatCompletionResponseChoice]
1264
    usage: UsageInfo
1265
    prompt_logprobs: Optional[list[Optional[dict[int, Logprob]]]] = None
1266
1267


1268
class DeltaMessage(OpenAIBaseModel):
1269
1270
    role: Optional[str] = None
    content: Optional[str] = None
1271
    reasoning_content: Optional[str] = None
1272
    tool_calls: list[DeltaToolCall] = Field(default_factory=list)
1273
1274


1275
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
1276
1277
    index: int
    delta: DeltaMessage
1278
    logprobs: Optional[ChatCompletionLogProbs] = None
1279
    finish_reason: Optional[str] = None
1280
    stop_reason: Optional[Union[int, str]] = None
1281
1282


1283
class ChatCompletionStreamResponse(OpenAIBaseModel):
1284
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
1285
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
1286
1287
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1288
    choices: list[ChatCompletionResponseStreamChoice]
1289
    usage: Optional[UsageInfo] = Field(default=None)
1290
1291


1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = None


class TranscriptionStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
    object: Literal["transcription.chunk"] = "transcription.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranscriptionResponseStreamChoice]
    usage: Optional[UsageInfo] = Field(default=None)


1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

1326
    # The parameters of the request.
1327
    body: Union[ChatCompletionRequest, EmbeddingRequest, ScoreRequest]
1328

1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
    @field_validator('body', mode='plain')
    @classmethod
    def check_type_for_url(cls, value: Any, info: ValidationInfo):
        # Use url to disambiguate models
        url = info.data['url']
        if url == "/v1/chat/completions":
            return ChatCompletionRequest.model_validate(value)
        if url == "/v1/embeddings":
            return TypeAdapter(EmbeddingRequest).validate_python(value)
        if url == "/v1/score":
            return ScoreRequest.model_validate(value)
        return TypeAdapter(Union[ChatCompletionRequest, EmbeddingRequest,
                                 ScoreRequest]).validate_python(value)

1343

1344
1345
1346
1347
1348
1349
1350
1351
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
1352
1353
    body: Optional[Union[ChatCompletionResponse, EmbeddingResponse,
                         ScoreResponse]] = None
1354
1355


1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

1367
    response: Optional[BatchResponseData]
1368
1369
1370
1371

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
    error: Optional[Any]
1372
1373


1374
class TokenizeCompletionRequest(OpenAIBaseModel):
1375
    model: Optional[str] = None
1376
1377
    prompt: str

1378
1379
1380
1381
1382
1383
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."),
    )
1384
1385
1386


class TokenizeChatRequest(OpenAIBaseModel):
1387
    model: Optional[str] = None
1388
    messages: list[ChatCompletionMessageParam]
1389

1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
    add_generation_prompt: bool = Field(
        default=True,
        description=
        ("If true, the generation prompt will be added to the chat template. "
         "This is a parameter used by chat template in tokenizer config of the "
         "model."),
    )
    continue_final_message: bool = Field(
        default=False,
        description=
        ("If this is set, the chat will be formatted so that the final "
         "message in the chat is open-ended, without any EOS tokens. The "
         "model will continue this message rather than starting a new one. "
         "This allows you to \"prefill\" part of the model's response for it. "
         "Cannot be used at the same time as `add_generation_prompt`."),
    )
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
            "default)."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."),
    )
1423
    chat_template_kwargs: Optional[dict[str, Any]] = Field(
1424
1425
1426
1427
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
1428
    mm_processor_kwargs: Optional[dict[str, Any]] = Field(
1429
1430
1431
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
1432

1433
1434
1435
1436
1437
1438
1439
1440
1441
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
        if data.get("continue_final_message") and data.get(
                "add_generation_prompt"):
            raise ValueError("Cannot set both `continue_final_message` and "
                             "`add_generation_prompt` to True.")
        return data

1442
1443

TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
1444
1445
1446
1447
1448


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
1449
    tokens: list[int]
1450
1451
1452


class DetokenizeRequest(OpenAIBaseModel):
1453
    model: Optional[str] = None
1454
    tokens: list[int]
1455
1456
1457
1458


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
1459
1460


1461
class LoadLoRAAdapterRequest(BaseModel):
1462
1463
1464
1465
    lora_name: str
    lora_path: str


1466
class UnloadLoRAAdapterRequest(BaseModel):
1467
1468
    lora_name: str
    lora_int_id: Optional[int] = Field(default=None)
1469
1470
1471
1472
1473
1474
1475
1476
1477


## Protocols for Audio
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json",
                                         "vtt"]


class TranscriptionRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
1478
    # https://platform.openai.com/docs/api-reference/audio/createTranscription
1479
1480
1481
1482
1483
1484
1485

    file: UploadFile
    """
    The audio file object (not file name) to transcribe, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

1486
    model: Optional[str] = None
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
    """ID of the model to use.
    """

    language: Optional[str] = None
    """The language of the input audio.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy and latency.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    ## TODO (varun) : Support if set to 0, certain thresholds are met !!
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """

1522
    timestamp_granularities: list[Literal["word", "segment"]] = Field(
1523
1524
1525
1526
1527
1528
1529
1530
1531
        alias="timestamp_granularities[]", default=[])
    """The timestamp granularities to populate for this transcription.

    `response_format` must be set `verbose_json` to use timestamp granularities.
    Either or both of these options are supported: `word`, or `segment`. Note:
    There is no additional latency for segment timestamps, but generating word
    timestamps incurs additional latency.
    """

1532
1533
1534
1535
1536
1537
1538
1539
1540
    stream: Optional[bool] = False
    """Custom field not present in the original OpenAI definition. When set, 
    it will enable output to be streamed in a similar fashion as the Chat
    Completion endpoint. 
    """
    # Flattened stream option to simplify form data.
    stream_include_usage: Optional[bool] = False
    stream_continuous_usage_stats: Optional[bool] = False

1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
    # Default sampling parameters for transcription requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "temperature": 0,
    }

    def to_sampling_params(
            self,
            default_max_tokens: int,
            default_sampling_params: Optional[dict] = None) -> SamplingParams:
        # TODO(#9845): remove max_tokens when field is removed from OpenAI API
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"])

        return SamplingParams.from_optional(temperature=temperature,
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
                                            max_tokens=max_tokens,
                                            output_kind=RequestOutputKind.DELTA
                                            if self.stream \
                                            else RequestOutputKind.FINAL_ONLY)

    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
            raise ValueError(
                "Stream options can only be defined when `stream=True`.")

        return data
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632


# Transcription response objects
class TranscriptionResponse(OpenAIBaseModel):
    text: str
    """The transcribed text."""


class TranscriptionWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranscriptionSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

    avg_logprob: float
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

    compression_ratio: float
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

    no_speech_prob: float
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

1633
    tokens: list[int]
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
    """Array of token IDs for the text content."""


class TranscriptionResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The transcribed text."""

1647
    segments: Optional[list[TranscriptionSegment]] = None
1648
1649
    """Segments of the transcribed text and their corresponding details."""

1650
    words: Optional[list[TranscriptionWord]] = None
1651
    """Extracted words and their corresponding timestamps."""