protocol.py 102 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import json
Zhuohan Li's avatar
Zhuohan Li committed
7
import time
8
from http import HTTPStatus
9
from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
Zhuohan Li's avatar
Zhuohan Li committed
10

11
import regex as re
12
import torch
13
from fastapi import HTTPException, UploadFile
14
from openai.types.chat.chat_completion_audio import (
15
16
17
    ChatCompletionAudio as OpenAIChatCompletionAudio,
)
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
18
19
20
21
22
from openai.types.responses import (
    ResponseCodeInterpreterCallCodeDeltaEvent,
    ResponseCodeInterpreterCallCodeDoneEvent,
    ResponseCodeInterpreterCallCompletedEvent,
    ResponseCodeInterpreterCallInProgressEvent,
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
    ResponseCodeInterpreterCallInterpretingEvent,
    ResponseContentPartAddedEvent,
    ResponseContentPartDoneEvent,
    ResponseFunctionToolCall,
    ResponseInputItemParam,
    ResponseOutputItem,
    ResponseOutputItemAddedEvent,
    ResponseOutputItemDoneEvent,
    ResponsePrompt,
    ResponseReasoningItem,
    ResponseReasoningTextDeltaEvent,
    ResponseReasoningTextDoneEvent,
    ResponseStatus,
    ResponseWebSearchCallCompletedEvent,
    ResponseWebSearchCallInProgressEvent,
    ResponseWebSearchCallSearchingEvent,
)
40
from openai.types.responses import (
41
42
43
    ResponseCompletedEvent as OpenAIResponseCompletedEvent,
)
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
44
from openai.types.responses import (
45
46
    ResponseInProgressEvent as OpenAIResponseInProgressEvent,
)
47
from openai.types.responses.response_reasoning_item import (
48
49
    Content as ResponseReasoningTextContent,
)
50
51
52
53
54

# Backward compatibility for OpenAI client versions
try:  # For older openai versions (< 1.100.0)
    from openai.types.responses import ResponseTextConfig
except ImportError:  # For newer openai versions (>= 1.100.0)
55
    from openai.types.responses import ResponseFormatTextConfig as ResponseTextConfig
56

57

58
from openai.types.responses.response import IncompleteDetails, ToolChoice
59
60
from openai.types.responses.tool import Tool
from openai.types.shared import Metadata, Reasoning
61
62
63
64
65
66
from pydantic import (
    BaseModel,
    ConfigDict,
    Field,
    TypeAdapter,
    ValidationInfo,
67
    field_serializer,
68
69
70
    field_validator,
    model_validator,
)
Zhuohan Li's avatar
Zhuohan Li committed
71

72
from vllm import envs
73
74
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam, make_tool_call_id
from vllm.entrypoints.score_utils import ScoreContentPartParam, ScoreMultiModalParam
75
from vllm.logger import init_logger
76
from vllm.logprobs import Logprob
77
from vllm.pooling_params import PoolingParams
78
79
80
81
82
83
from vllm.sampling_params import (
    BeamSearchParams,
    RequestOutputKind,
    SamplingParams,
    StructuredOutputsParams,
)
84
from vllm.utils import random_uuid, resolve_obj_by_qualname
85

86
87
logger = init_logger(__name__)

88
_LONG_INFO = torch.iinfo(torch.long)
89

Zhuohan Li's avatar
Zhuohan Li committed
90

91
class OpenAIBaseModel(BaseModel):
92
93
94
    # OpenAI API does allow extra fields
    model_config = ConfigDict(extra="allow")

95
    # Cache class field names
96
    field_names: ClassVar[set[str] | None] = None
97

98
    @model_validator(mode="wrap")
99
    @classmethod
100
101
102
103
    def __log_extra_fields__(cls, data, handler):
        result = handler(data)
        if not isinstance(data, dict):
            return result
104
105
        field_names = cls.field_names
        if field_names is None:
106
107
108
109
            # Get all class field names and their potential aliases
            field_names = set()
            for field_name, field in cls.model_fields.items():
                field_names.add(field_name)
110
                if alias := getattr(field, "alias", None):
111
112
113
114
115
116
                    field_names.add(alias)
            cls.field_names = field_names

        # Compare against both field names and aliases
        if any(k not in field_names for k in data):
            logger.warning(
117
                "The following fields were present in the request but ignored: %s",
118
119
                data.keys() - field_names,
            )
120
        return result
121
122


123
class ErrorInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
124
125
    message: str
    type: str
126
    param: str | None = None
127
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
128
129


130
131
132
133
class ErrorResponse(OpenAIBaseModel):
    error: ErrorInfo


134
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
135
136
137
138
139
140
141
142
143
144
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
145
    group: str | None = None
146
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
147
148


149
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
150
151
152
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
153
    owned_by: str = "vllm"
154
155
156
    root: str | None = None
    parent: str | None = None
    max_model_len: int | None = None
157
    permission: list[ModelPermission] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
158
159


160
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
161
    object: str = "list"
162
    data: list[ModelCard] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
163
164


165
class PromptTokenUsageInfo(OpenAIBaseModel):
166
    cached_tokens: int | None = None
167
168


169
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
170
171
    prompt_tokens: int = 0
    total_tokens: int = 0
172
173
    completion_tokens: int | None = 0
    prompt_tokens_details: PromptTokenUsageInfo | None = None
Zhuohan Li's avatar
Zhuohan Li committed
174
175


176
177
class RequestResponseMetadata(BaseModel):
    request_id: str
178
    final_usage_info: UsageInfo | None = None
179
180


181
182
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
183
    description: str | None = None
184
185
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
186
187
    json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
    strict: bool | None = None
188
189


190
191
192
193
class StructuralTag(OpenAIBaseModel):
    begin: str
    # schema is the field, but that causes conflicts with pydantic so
    # instead use structural_tag_schema with an alias
194
    structural_tag_schema: dict[str, Any] | None = Field(default=None, alias="schema")
195
196
197
198
199
200
201
202
203
    end: str


class StructuralTagResponseFormat(OpenAIBaseModel):
    type: Literal["structural_tag"]
    structures: list[StructuralTag]
    triggers: list[str]


204
class ResponseFormat(OpenAIBaseModel):
205
    # type must be "json_schema", "json_object", or "text"
206
    type: Literal["text", "json_object", "json_schema"]
207
    json_schema: JsonSchemaResponseFormat | None = None
208
209


210
AnyResponseFormat: TypeAlias = ResponseFormat | StructuralTagResponseFormat
211
212


213
class StreamOptions(OpenAIBaseModel):
214
215
    include_usage: bool | None = True
    continuous_usage_stats: bool | None = False
216
217


218
219
class FunctionDefinition(OpenAIBaseModel):
    name: str
220
221
    description: str | None = None
    parameters: dict[str, Any] | None = None
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


238
239
# extra="forbid" is a workaround to have kwargs as a field,
# see https://github.com/pydantic/pydantic/issues/3125
240
241
class LogitsProcessorConstructor(BaseModel):
    qualname: str
242
243
    args: list[Any] | None = None
    kwargs: dict[str, Any] | None = None
244

245
246
    model_config = ConfigDict(extra="forbid")

247

248
LogitsProcessors = list[str | LogitsProcessorConstructor]
249
250


251
def get_logits_processors(
252
253
    processors: LogitsProcessors | None, pattern: str | None
) -> list[Any] | None:
254
255
256
    if processors and pattern:
        logits_processors = []
        for processor in processors:
257
            qualname = processor if isinstance(processor, str) else processor.qualname
258
259
260
261
            if not re.match(pattern, qualname):
                raise ValueError(
                    f"Logits processor '{qualname}' is not allowed by this "
                    "server. See --logits-processor-pattern engine argument "
262
263
                    "for more information."
                )
264
265
266
267
268
269
270
            try:
                logits_processor = resolve_obj_by_qualname(qualname)
            except Exception as e:
                raise ValueError(
                    f"Logits processor '{qualname}' could not be resolved: {e}"
                ) from e
            if isinstance(processor, LogitsProcessorConstructor):
271
272
273
                logits_processor = logits_processor(
                    *processor.args or [], **processor.kwargs or {}
                )
274
275
276
277
278
            logits_processors.append(logits_processor)
        return logits_processors
    elif processors:
        raise ValueError(
            "The `logits_processors` argument is not supported by this "
279
            "server. See --logits-processor-pattern engine argument "
280
281
            "for more information."
        )
282
283
284
    return None


285
286
287
ResponseInputOutputItem: TypeAlias = (
    ResponseInputItemParam | ResponseReasoningItem | ResponseFunctionToolCall
)
288
289


290
291
292
class ResponsesRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/responses/create
293
294
    background: bool | None = False
    include: (
295
296
297
298
299
300
301
302
303
304
        list[
            Literal[
                "code_interpreter_call.outputs",
                "computer_call_output.output.image_url",
                "file_search_call.results",
                "message.input_image.image_url",
                "message.output_text.logprobs",
                "reasoning.encrypted_content",
            ],
        ]
305
306
307
308
309
310
311
312
313
314
315
316
        | None
    ) = None
    input: str | list[ResponseInputOutputItem]
    instructions: str | None = None
    max_output_tokens: int | None = None
    max_tool_calls: int | None = None
    metadata: Metadata | None = None
    model: str | None = None
    parallel_tool_calls: bool | None = True
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
317
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto"
318
319
320
321
    store: bool | None = True
    stream: bool | None = False
    temperature: float | None = None
    text: ResponseTextConfig | None = None
322
323
    tool_choice: ToolChoice = "auto"
    tools: list[Tool] = Field(default_factory=list)
324
325
326
327
    top_logprobs: int | None = 0
    top_p: float | None = None
    truncation: Literal["auto", "disabled"] | None = "disabled"
    user: str | None = None
328
329
330
331
332
333
334

    # --8<-- [start:responses-extra-params]
    request_id: str = Field(
        default_factory=lambda: f"resp_{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
335
336
            "through out the inference process and return in response."
        ),
337
    )
338
    mm_processor_kwargs: dict[str, Any] | None = Field(
339
340
341
342
343
344
345
346
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
347
348
            "if the served model does not use priority scheduling."
        ),
349
    )
350
    cache_salt: str | None = Field(
351
352
353
354
355
356
357
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
358
359
360
            "to 256 bit). Not supported by vLLM engine V0."
        ),
    )
361
362
363
364
365
366

    enable_response_messages: bool = Field(
        default=False,
        description=(
            "Dictates whether or not to return messages as part of the "
            "response object. Currently only supported for non-streaming "
367
368
369
            "non-background and gpt-oss only. "
        ),
    )
370
371
372
373
374
375
376
377
378
379
    # --8<-- [end:responses-extra-params]

    _DEFAULT_SAMPLING_PARAMS = {
        "temperature": 1.0,
        "top_p": 1.0,
    }

    def to_sampling_params(
        self,
        default_max_tokens: int,
380
        default_sampling_params: dict | None = None,
381
382
383
384
385
386
387
388
389
    ) -> SamplingParams:
        if self.max_output_tokens is None:
            max_tokens = default_max_tokens
        else:
            max_tokens = min(self.max_output_tokens, default_max_tokens)

        default_sampling_params = default_sampling_params or {}
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
390
391
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
392
393
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
394
395
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
396
        stop_token_ids = default_sampling_params.get("stop_token_ids")
397
398

        # Structured output
399
        structured_outputs = None
400
401
        if self.text is not None and self.text.format is not None:
            response_format = self.text.format
402
403
404
405
            if (
                response_format.type == "json_schema"
                and response_format.schema_ is not None
            ):
406
                structured_outputs = StructuredOutputsParams(
407
408
                    json=response_format.schema_
                )
409
410
411
412
413
414
415
416
            elif response_format.type == "json_object":
                raise NotImplementedError("json_object is not supported")

        # TODO: add more parameters
        return SamplingParams.from_optional(
            temperature=temperature,
            top_p=top_p,
            max_tokens=max_tokens,
417
            logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
418
            stop_token_ids=stop_token_ids,
419
420
421
            output_kind=(
                RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
            ),
422
            structured_outputs=structured_outputs,
423
424
        )

425
426
427
428
    def is_include_output_logprobs(self) -> bool:
        """Check if the request includes output logprobs."""
        if self.include is None:
            return False
429
430
431
432
        return (
            isinstance(self.include, list)
            and "message.output_text.logprobs" in self.include
        )
433

434
435
436
437
438
    @model_validator(mode="before")
    def validate_background(cls, data):
        if not data.get("background"):
            return data
        if not data.get("store", True):
439
            raise ValueError("background can only be used when `store` is true")
440
441
442
443
444
445
446
447
        return data

    @model_validator(mode="before")
    def validate_prompt(cls, data):
        if data.get("prompt") is not None:
            raise ValueError("prompt template is not supported")
        return data

448
449
450
451
452
453
    @model_validator(mode="before")
    def check_cache_salt_support(cls, data):
        if data.get("cache_salt") is not None:
            if not envs.VLLM_USE_V1:
                raise ValueError(
                    "Parameter 'cache_salt' is not supported with "
454
455
456
457
458
459
                    "this instance of vLLM, which uses engine V0."
                )
            if not isinstance(data["cache_salt"], str) or not data["cache_salt"]:
                raise ValueError(
                    "Parameter 'cache_salt' must be a non-empty string if provided."
                )
460
461
        return data

462

463
class ChatCompletionRequest(OpenAIBaseModel):
464
465
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
466
    messages: list[ChatCompletionMessageParam]
467
468
469
470
471
472
    model: str | None = None
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: bool | None = False
    top_logprobs: int | None = 0
    max_tokens: int | None = Field(
473
        default=None,
474
475
        deprecated="max_tokens is deprecated in favor of "
        "the max_completion_tokens field",
476
    )
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
    max_completion_tokens: int | None = None
    n: int | None = 1
    presence_penalty: float | None = 0.0
    response_format: AnyResponseFormat | None = None
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    temperature: float | None = None
    top_p: float | None = None
    tools: list[ChatCompletionToolsParam] | None = None
    tool_choice: (
        Literal["none"]
        | Literal["auto"]
        | Literal["required"]
        | ChatCompletionNamedToolChoiceParam
        | None
    ) = "none"
    reasoning_effort: Literal["low", "medium", "high"] | None = None
496
    include_reasoning: bool = True
497

498
    # NOTE this will be ignored by vLLM -- the model determines the behavior
499
500
    parallel_tool_calls: bool | None = False
    user: str | None = None
501

502
    # --8<-- [start:chat-completion-sampling-params]
503
    best_of: int | None = None
504
    use_beam_search: bool = False
505
506
507
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
508
    length_penalty: float = 1.0
509
    stop_token_ids: list[int] | None = []
510
511
512
513
514
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
515
516
517
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    prompt_logprobs: int | None = None
    allowed_token_ids: list[int] | None = None
518
    bad_words: list[str] = Field(default_factory=list)
519
    # --8<-- [end:chat-completion-sampling-params]
520

521
    # --8<-- [start:chat-completion-extra-params]
522
    echo: bool = Field(
523
524
525
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
526
527
            "if they belong to the same role."
        ),
528
    )
529
    add_generation_prompt: bool = Field(
530
        default=True,
531
532
533
534
535
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
536
    )
537
538
    continue_final_message: bool = Field(
        default=False,
539
540
541
542
543
544
545
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
546
    )
547
    add_special_tokens: bool = Field(
548
549
550
551
552
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
553
            "special tokens so this should be set to false (as is the "
554
555
            "default)."
        ),
556
    )
557
    documents: list[dict[str, str]] | None = Field(
558
        default=None,
559
560
561
562
563
564
565
        description=(
            "A list of dicts representing documents that will be accessible to "
            "the model if it is performing RAG (retrieval-augmented generation)."
            " If the template does not support RAG, this argument will have no "
            "effect. We recommend that each document should be a dict containing "
            '"title" and "text" keys.'
        ),
566
    )
567
    chat_template: str | None = Field(
568
569
570
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
571
572
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
573
574
            "does not define one."
        ),
575
    )
576
    chat_template_kwargs: dict[str, Any] | None = Field(
577
        default=None,
578
579
        description=(
            "Additional keyword args to pass to the template renderer. "
580
581
            "Will be accessible by the chat template."
        ),
582
    )
583
    mm_processor_kwargs: dict[str, Any] | None = Field(
584
585
586
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
587
    structured_outputs: StructuredOutputsParams | None = Field(
588
        default=None,
589
        description="Additional kwargs for structured outputs",
590
    )
591
    guided_json: str | dict | BaseModel | None = Field(
592
593
594
595
        default=None,
        description=(
            "`guided_json` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
596
597
            "Please pass `json` to `structured_outputs` instead."
        ),
598
    )
599
    guided_regex: str | None = Field(
600
601
602
603
        default=None,
        description=(
            "`guided_regex` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
604
605
            "Please pass `regex` to `structured_outputs` instead."
        ),
606
    )
607
    guided_choice: list[str] | None = Field(
608
609
610
611
        default=None,
        description=(
            "`guided_choice` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
612
613
            "Please pass `choice` to `structured_outputs` instead."
        ),
614
    )
615
    guided_grammar: str | None = Field(
616
617
618
619
        default=None,
        description=(
            "`guided_grammar` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
620
621
            "Please pass `grammar` to `structured_outputs` instead."
        ),
622
    )
623
    structural_tag: str | None = Field(
624
625
626
627
        default=None,
        description=(
            "`structural_tag` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
628
629
            "Please pass `structural_tag` to `structured_outputs` instead."
        ),
630
    )
631
    guided_decoding_backend: str | None = Field(
632
633
634
635
        default=None,
        description=(
            "`guided_decoding_backend` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
636
637
            "Please remove it from your request."
        ),
638
    )
639
    guided_whitespace_pattern: str | None = Field(
640
641
642
643
644
645
646
        default=None,
        description=(
            "`guided_whitespace_pattern` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
            "Please pass `whitespace_pattern` to `structured_outputs` instead."
        ),
    )
647
648
649
650
651
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
652
653
            "if the served model does not use priority scheduling."
        ),
654
    )
655
656
657
658
659
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
660
661
            "through out the inference process and return in response."
        ),
662
    )
663
    logits_processors: LogitsProcessors | None = Field(
664
665
666
667
668
669
670
671
672
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
673
674
675
            "{'param': 'value'}}."
        ),
    )
676
    return_tokens_as_token_ids: bool | None = Field(
677
678
679
680
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
681
682
683
            "that are not JSON-encodable can be identified."
        ),
    )
684
    return_token_ids: bool | None = Field(
685
686
687
688
689
690
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
691
692
693
            "need to map generated text back to input tokens."
        ),
    )
694
    cache_salt: str | None = Field(
695
696
697
698
699
700
701
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
702
703
704
            "to 256 bit). Not supported by vLLM engine V0."
        ),
    )
705
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
706
        default=None,
707
708
        description="KVTransfer parameters used for disaggregated serving.",
    )
709

710
    vllm_xargs: dict[str, str | int | float] | None = Field(
711
        default=None,
712
713
714
715
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
716
717
    )

718
    # --8<-- [end:chat-completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
719

720
721
722
723
724
    # Default sampling parameters for chat completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
725
        "top_k": 0,
726
727
728
729
        "min_p": 0.0,
    }

    def to_beam_search_params(
730
731
        self, max_tokens: int, default_sampling_params: dict
    ) -> BeamSearchParams:
732
        n = self.n if self.n is not None else 1
733
734
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
735
736
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
737
738
739
740
741
742

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
743
            length_penalty=self.length_penalty,
744
745
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
746

747
    def to_sampling_params(
748
        self,
749
        max_tokens: int,
750
        logits_processor_pattern: str | None,
751
        default_sampling_params: dict,
752
    ) -> SamplingParams:
753
754
755
756
757
758
759
760
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
761
762
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
763
764
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
765
766
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
767
768
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
769
770
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
771
772
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
773
774
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
775

776
777
778
779
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

780
781
782
783
784
785
786
787
788
789
790
791
792
793
        # Forward deprecated guided_* parameters to structured_outputs
        if self.structured_outputs is None:
            kwargs = dict[str, Any](
                json=self.guided_json,
                regex=self.guided_regex,
                choice=self.guided_choice,
                grammar=self.guided_grammar,
                whitespace_pattern=self.guided_whitespace_pattern,
                structural_tag=self.structural_tag,
            )
            kwargs = {k: v for k, v in kwargs.items() if v is not None}
            if len(kwargs) > 0:
                self.structured_outputs = StructuredOutputsParams(**kwargs)

794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
        response_format = self.response_format
        json_schema_from_tool = self._get_json_schema_from_tool()
        if response_format is not None or json_schema_from_tool is not None:
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
            if response_format is not None:
                if response_format.type == "json_object":
                    self.structured_outputs.json_object = True
                elif response_format.type == "json_schema":
                    json_schema = response_format.json_schema
                    assert json_schema is not None
                    self.structured_outputs.json = json_schema.json_schema
                elif response_format.type == "structural_tag":
                    structural_tag = response_format
                    assert structural_tag is not None and isinstance(
813
814
                        structural_tag, StructuralTagResponseFormat
                    )
815
                    s_tag_obj = structural_tag.model_dump(by_alias=True)
816
                    self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
817
818
819
820

            # Set structured output params for tool calling
            if json_schema_from_tool is not None:
                self.structured_outputs.json = json_schema_from_tool
821

822
823
824
825
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
826
        return SamplingParams.from_optional(
827
            n=self.n,
828
            best_of=self.best_of,
829
830
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
831
832
833
834
835
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
836
            seed=self.seed,
837
838
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
839
            logprobs=self.top_logprobs if self.logprobs else None,
840
            prompt_logprobs=prompt_logprobs,
841
            ignore_eos=self.ignore_eos,
842
            max_tokens=max_tokens,
843
            min_tokens=self.min_tokens,
844
845
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
846
847
848
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
849
            include_stop_str_in_output=self.include_stop_str_in_output,
850
            truncate_prompt_tokens=self.truncate_prompt_tokens,
851
852
853
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
854
            structured_outputs=self.structured_outputs,
Robert Shaw's avatar
Robert Shaw committed
855
            logit_bias=self.logit_bias,
856
            bad_words=self.bad_words,
857
            allowed_token_ids=self.allowed_token_ids,
858
859
            extra_args=extra_args or None,
        )
860

861
    def _get_json_schema_from_tool(self) -> str | dict | None:
862
863
864
865
866
867
868
869
870
        # user has chosen to not use any tool
        if self.tool_choice == "none" or self.tools is None:
            return None

        # user has chosen to use a named tool
        if type(self.tool_choice) is ChatCompletionNamedToolChoiceParam:
            tool_name = self.tool_choice.function.name
            tools = {tool.function.name: tool.function for tool in self.tools}
            if tool_name not in tools:
871
                raise ValueError(f"Tool '{tool_name}' has not been passed in `tools`.")
872
873
874
            tool = tools[tool_name]
            return tool.parameters

875
876
877
878
879
880
881
882
        if self.tool_choice == "required":
            # Pydantic schema generation cannot be used since the JSON schema
            # has to be constructed for a specific instantiation of a tool list
            # so that parameters of a function are correctly generated
            # based on the chosen function name
            def get_tool_schema(tool: ChatCompletionToolsParam) -> dict:
                return {
                    "properties": {
883
                        "name": {"type": "string", "enum": [tool.function.name]},
884
885
886
887
888
                        # parameters are always generated as '{}' in the final
                        # output if they are missing from the request
                        # (i.e. are None or '{}') so the schema is
                        # updated to produce an empty object in that case
                        "parameters": tool.function.parameters
889
890
                        if tool.function.parameters
                        else {"type": "object", "properties": {}},
891
                    },
892
                    "required": ["name", "parameters"],
893
894
                }

895
            def get_tool_schema_defs(tools: list[ChatCompletionToolsParam]) -> dict:
896
897
898
899
900
901
                all_defs = dict[str, dict[str, Any]]()
                for tool in tools:
                    if tool.function.parameters is None:
                        continue
                    defs = tool.function.parameters.pop("$defs", {})
                    for def_name, def_schema in defs.items():
902
                        if def_name in all_defs and all_defs[def_name] != def_schema:
903
904
905
                            raise ValueError(
                                f"Tool definition '{def_name}' has "
                                "multiple schemas, which is not "
906
907
                                "supported."
                            )
908
909
910
911
                        else:
                            all_defs[def_name] = def_schema
                return all_defs

912
913
914
915
916
            json_schema = {
                "type": "array",
                "minItems": 1,
                "items": {
                    "type": "object",
917
918
                    "anyOf": [get_tool_schema(tool) for tool in self.tools],
                },
919
            }
920
921
922
            json_schema_defs = get_tool_schema_defs(self.tools)
            if json_schema_defs:
                json_schema["$defs"] = json_schema_defs
923
924
            return json_schema

925
        return None
926

927
    @model_validator(mode="before")
928
    @classmethod
929
930
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
931
            raise ValueError("Stream options can only be defined when `stream=True`.")
932
933
934
935
936
937
938

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
939
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
940
                raise ValueError(
941
942
                    "`prompt_logprobs` are not available when `stream=True`."
                )
943

944
            if prompt_logprobs < 0 and prompt_logprobs != -1:
945
                raise ValueError("`prompt_logprobs` must be a positive value or -1.")
946
            if prompt_logprobs == -1 and not envs.VLLM_USE_V1:
947
948
949
                raise ValueError(
                    "`prompt_logprobs=-1` is only supported with vLLM engine V1."
                )
950
        if (top_logprobs := data.get("top_logprobs")) is not None:
951
            if top_logprobs < 0 and top_logprobs != -1:
952
                raise ValueError("`top_logprobs` must be a positive value or -1.")
953

954
            if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"):
955
956
957
958
959
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )

        return data
960

961
962
    @model_validator(mode="before")
    @classmethod
963
    def check_structured_outputs_count(cls, data):
964
965
966
        if isinstance(data, ValueError):
            raise data

967
        if data.get("structured_outputs", None) is None:
968
969
            return data

970
        structured_outputs_kwargs = data["structured_outputs"]
971
972
        count = sum(
            structured_outputs_kwargs.get(k) is not None
973
974
            for k in ("json", "regex", "choice")
        )
975
976
        # you can only use one kind of constraints for structured outputs
        if count > 1:
977
            raise ValueError(
978
                "You can only use one kind of constraints for structured "
979
980
                "outputs ('json', 'regex' or 'choice')."
            )
981
982
        # you can only either use structured outputs or tools, not both
        if count > 1 and data.get("tool_choice", "none") not in (
983
984
985
            "none",
            "auto",
            "required",
986
        ):
987
            raise ValueError(
988
                "You can only either use constraints for structured outputs "
989
990
                "or tools, not both."
            )
991
992
993
994
        return data

    @model_validator(mode="before")
    @classmethod
995
996
997
    def check_tool_usage(cls, data):
        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
998
        if "tool_choice" not in data and data.get("tools"):
999
1000
            data["tool_choice"] = "auto"

1001
        # if "tool_choice" is "none" -- no validation is needed for tools
1002
1003
1004
        if "tool_choice" in data and data["tool_choice"] == "none":
            return data

1005
        # if "tool_choice" is specified -- validation
1006
        if "tool_choice" in data and data["tool_choice"] is not None:
1007
            # ensure that if "tool choice" is specified, tools are present
1008
            if "tools" not in data or data["tools"] is None:
1009
                raise ValueError("When using `tool_choice`, `tools` must be set.")
1010
1011

            # make sure that tool choice is either a named tool
1012
            # OR that it's set to "auto" or "required"
1013
1014
1015
            if data["tool_choice"] not in ["auto", "required"] and not isinstance(
                data["tool_choice"], dict
            ):
1016
                raise ValueError(
1017
1018
1019
                    f"Invalid value for `tool_choice`: {data['tool_choice']}! "
                    'Only named tools, "none", "auto" or "required" '
                    "are supported."
1020
                )
1021

1022
1023
1024
            # if tool_choice is "required" but the "tools" list is empty,
            # override the data to behave like "none" to align with
            # OpenAI’s behavior.
1025
1026
1027
1028
1029
            if (
                data["tool_choice"] == "required"
                and isinstance(data["tools"], list)
                and len(data["tools"]) == 0
            ):
1030
1031
1032
1033
                data["tool_choice"] = "none"
                del data["tools"]
                return data

1034
1035
            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
1036
1037
            correct_usage_message = (
                'Correct usage: `{"type": "function",'
1038
                ' "function": {"name": "my_function"}}`'
1039
            )
1040
1041
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
1042
1043
                function = data["tool_choice"].get("function")
                if not isinstance(function, dict):
1044
                    raise ValueError(
1045
                        f"Invalid value for `function`: `{function}` in "
1046
1047
                        f"`tool_choice`! {correct_usage_message}"
                    )
1048
                if "name" not in function:
1049
1050
1051
1052
                    raise ValueError(
                        f"Expected field `name` in `function` in "
                        f"`tool_choice`! {correct_usage_message}"
                    )
1053
                function_name = function["name"]
1054
                if not isinstance(function_name, str) or len(function_name) == 0:
1055
                    raise ValueError(
1056
                        f"Invalid `name` in `function`: `{function_name}`"
1057
1058
                        f" in `tool_choice`! {correct_usage_message}"
                    )
1059
                for tool in data["tools"]:
1060
                    if tool["function"]["name"] == function_name:
1061
1062
1063
1064
1065
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
1066
1067
                        " of the specified `tools`"
                    )
1068
1069
        return data

1070
1071
1072
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
1073
1074
1075
1076
1077
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
1078
1079
        return data

1080
1081
1082
1083
1084
1085
1086
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
        if data.get("cache_salt") is not None:
            if not envs.VLLM_USE_V1:
                raise ValueError(
                    "Parameter 'cache_salt' is not supported with "
1087
1088
1089
1090
1091
1092
                    "this instance of vLLM, which uses engine V0."
                )
            if not isinstance(data["cache_salt"], str) or not data["cache_salt"]:
                raise ValueError(
                    "Parameter 'cache_salt' must be a non-empty string if provided."
                )
1093
1094
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1095

1096
class CompletionRequest(OpenAIBaseModel):
1097
1098
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
1099
1100
1101
1102
1103
1104
1105
1106
    model: str | None = None
    prompt: list[int] | list[list[int]] | str | list[str] | None = None
    best_of: int | None = None
    echo: bool | None = False
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: int | None = None
    max_tokens: int | None = 16
1107
    n: int = 1
1108
1109
1110
1111
1112
1113
1114
1115
1116
    presence_penalty: float | None = 0.0
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    suffix: str | None = None
    temperature: float | None = None
    top_p: float | None = None
    user: str | None = None
1117

1118
    # --8<-- [start:completion-sampling-params]
1119
    use_beam_search: bool = False
1120
1121
1122
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
1123
    length_penalty: float = 1.0
1124
    stop_token_ids: list[int] | None = []
1125
1126
1127
1128
1129
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
1130
1131
1132
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    allowed_token_ids: list[int] | None = None
    prompt_logprobs: int | None = None
1133
    # --8<-- [end:completion-sampling-params]
1134

1135
    # --8<-- [start:completion-extra-params]
1136
    prompt_embeds: bytes | list[bytes] | None = None
1137
1138
    add_special_tokens: bool = Field(
        default=True,
1139
        description=(
1140
            "If true (the default), special tokens (e.g. BOS) will be added to "
1141
1142
            "the prompt."
        ),
1143
    )
1144
    response_format: AnyResponseFormat | None = Field(
1145
        default=None,
1146
1147
1148
1149
1150
        description=(
            "Similar to chat completion, this parameter specifies the format "
            "of output. Only {'type': 'json_object'}, {'type': 'json_schema'}"
            ", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
        ),
1151
    )
1152
    structured_outputs: StructuredOutputsParams | None = Field(
1153
        default=None,
1154
        description="Additional kwargs for structured outputs",
1155
    )
1156
    guided_json: str | dict | BaseModel | None = Field(
1157
1158
1159
1160
        default=None,
        description=(
            "`guided_json` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
1161
1162
            "Please pass `json` to `structured_outputs` instead."
        ),
1163
    )
1164
    guided_regex: str | None = Field(
1165
1166
1167
1168
        default=None,
        description=(
            "`guided_regex` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
1169
1170
            "Please pass `regex` to `structured_outputs` instead."
        ),
1171
    )
1172
    guided_choice: list[str] | None = Field(
1173
1174
1175
1176
        default=None,
        description=(
            "`guided_choice` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
1177
1178
            "Please pass `choice` to `structured_outputs` instead."
        ),
1179
    )
1180
    guided_grammar: str | None = Field(
1181
1182
1183
1184
        default=None,
        description=(
            "`guided_grammar` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
1185
1186
            "Please pass `grammar` to `structured_outputs` instead."
        ),
1187
    )
1188
    guided_decoding_backend: str | None = Field(
1189
1190
1191
1192
        default=None,
        description=(
            "`guided_decoding_backend` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
1193
1194
            "Please remove it from your request."
        ),
1195
    )
1196
    guided_whitespace_pattern: str | None = Field(
1197
1198
1199
1200
1201
1202
1203
        default=None,
        description=(
            "`guided_whitespace_pattern` is deprecated. "
            "This will be removed in v0.12.0 or v1.0.0, whichever is soonest. "
            "Please pass `whitespace_pattern` to `structured_outputs` instead."
        ),
    )
1204
1205
1206
1207
1208
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1209
1210
            "if the served model does not use priority scheduling."
        ),
1211
    )
1212
1213
1214
1215
1216
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1217
1218
            "through out the inference process and return in response."
        ),
1219
    )
1220
    logits_processors: LogitsProcessors | None = Field(
1221
1222
1223
1224
1225
1226
1227
1228
1229
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
1230
1231
1232
            "{'param': 'value'}}."
        ),
    )
1233

1234
    return_tokens_as_token_ids: bool | None = Field(
1235
1236
1237
1238
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
1239
1240
1241
            "that are not JSON-encodable can be identified."
        ),
    )
1242
    return_token_ids: bool | None = Field(
1243
1244
1245
1246
1247
1248
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
1249
1250
1251
            "need to map generated text back to input tokens."
        ),
    )
1252

1253
    cache_salt: str | None = Field(
1254
1255
1256
1257
1258
1259
1260
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
1261
1262
1263
            "to 256 bit). Not supported by vLLM engine V0."
        ),
    )
1264

1265
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
1266
        default=None,
1267
1268
        description="KVTransfer parameters used for disaggregated serving.",
    )
Robert Shaw's avatar
Robert Shaw committed
1269

1270
    vllm_xargs: dict[str, str | int | float] | None = Field(
1271
        default=None,
1272
1273
1274
1275
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
1276
1277
    )

1278
    # --8<-- [end:completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
1279

1280
1281
1282
1283
1284
    # Default sampling parameters for completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
1285
        "top_k": 0,
1286
1287
1288
1289
        "min_p": 0.0,
    }

    def to_beam_search_params(
1290
1291
        self,
        max_tokens: int,
1292
        default_sampling_params: dict | None = None,
1293
1294
1295
    ) -> BeamSearchParams:
        if default_sampling_params is None:
            default_sampling_params = {}
1296
        n = self.n if self.n is not None else 1
1297
1298
1299

        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get("temperature", 1.0)
1300
1301
1302
1303
1304
1305

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
1306
            length_penalty=self.length_penalty,
1307
1308
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
1309

1310
    def to_sampling_params(
1311
        self,
1312
        max_tokens: int,
1313
1314
        logits_processor_pattern: str | None,
        default_sampling_params: dict | None = None,
1315
    ) -> SamplingParams:
1316
1317
        if default_sampling_params is None:
            default_sampling_params = {}
1318

1319
1320
1321
1322
1323
1324
1325
1326
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
1327
1328
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
1329
1330
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
1331
1332
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
1333
1334
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
1335
1336
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
1337
1338
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
1339
1340
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
1341

1342
1343
1344
1345
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

1346
1347
        echo_without_generation = self.echo and self.max_tokens == 0

1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
        # Forward deprecated guided_* parameters to structured_outputs
        if self.structured_outputs is None:
            kwargs = dict[str, Any](
                json=self.guided_json,
                regex=self.guided_regex,
                choice=self.guided_choice,
                grammar=self.guided_grammar,
                whitespace_pattern=self.guided_whitespace_pattern,
            )
            kwargs = {k: v for k, v in kwargs.items() if v is not None}
            if len(kwargs) > 0:
                self.structured_outputs = StructuredOutputsParams(**kwargs)

1361
1362
1363
1364
1365
        if (
            self.structured_outputs is not None
            and self.response_format is not None
            and self.response_format.type == "json_object"
        ):
1366
            self.structured_outputs.json_object = True
1367

1368
1369
1370
1371
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
1372
        return SamplingParams.from_optional(
1373
            n=self.n,
1374
            best_of=self.best_of,
1375
1376
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
1377
1378
1379
1380
1381
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
1382
            seed=self.seed,
1383
1384
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
1385
            logprobs=self.logprobs,
1386
            ignore_eos=self.ignore_eos,
1387
            max_tokens=max_tokens if not echo_without_generation else 1,
1388
            min_tokens=self.min_tokens,
1389
            prompt_logprobs=prompt_logprobs,
1390
            skip_special_tokens=self.skip_special_tokens,
1391
            spaces_between_special_tokens=self.spaces_between_special_tokens,
1392
            include_stop_str_in_output=self.include_stop_str_in_output,
1393
1394
1395
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
1396
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1397
1398
1399
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
1400
            structured_outputs=self.structured_outputs,
1401
            logit_bias=self.logit_bias,
Robert Shaw's avatar
Robert Shaw committed
1402
            allowed_token_ids=self.allowed_token_ids,
1403
            extra_args=extra_args or None,
1404
        )
1405

1406
1407
    @model_validator(mode="before")
    @classmethod
1408
    def check_structured_outputs_count(cls, data):
1409
        if data.get("structured_outputs", None) is None:
1410
1411
            return data

1412
        structured_outputs_kwargs = data["structured_outputs"]
1413
1414
        count = sum(
            structured_outputs_kwargs.get(k) is not None
1415
1416
            for k in ("json", "regex", "choice")
        )
1417
        if count > 1:
1418
            raise ValueError(
1419
                "You can only use one kind of constraints for structured "
1420
1421
                "outputs ('json', 'regex' or 'choice')."
            )
1422
1423
        return data

1424
1425
1426
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
1427
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
1428
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
1429
                raise ValueError(
1430
1431
                    "`prompt_logprobs` are not available when `stream=True`."
                )
1432

1433
            if prompt_logprobs < 0 and prompt_logprobs != -1:
1434
                raise ValueError("`prompt_logprobs` must be a positive value or -1.")
1435
            if prompt_logprobs == -1 and not envs.VLLM_USE_V1:
1436
1437
1438
                raise ValueError(
                    "`prompt_logprobs=-1` is only supported with vLLM engine V1."
                )
1439
1440
1441
        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
            raise ValueError("`logprobs` must be a positive value.")

1442
1443
        return data

1444
1445
1446
1447
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
1448
            raise ValueError("Stream options can only be defined when `stream=True`.")
1449

1450
1451
        return data

1452
1453
1454
    @model_validator(mode="before")
    @classmethod
    def validate_prompt_and_prompt_embeds(cls, data):
1455
1456
1457
        prompt = data.get("prompt")
        prompt_embeds = data.get("prompt_embeds")

1458
1459
1460
1461
        prompt_is_empty = prompt is None or (isinstance(prompt, str) and prompt == "")
        embeds_is_empty = prompt_embeds is None or (
            isinstance(prompt_embeds, list) and len(prompt_embeds) == 0
        )
1462
1463

        if prompt_is_empty and embeds_is_empty:
1464
            raise ValueError(
1465
1466
1467
                "Either prompt or prompt_embeds must be provided and non-empty."
            )

1468
1469
        return data

1470
1471
1472
1473
1474
1475
1476
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
        if data.get("cache_salt") is not None:
            if not envs.VLLM_USE_V1:
                raise ValueError(
                    "Parameter 'cache_salt' is not supported with "
1477
1478
1479
1480
1481
1482
                    "this instance of vLLM, which uses engine V0."
                )
            if not isinstance(data["cache_salt"], str) or not data["cache_salt"]:
                raise ValueError(
                    "Parameter 'cache_salt' must be a non-empty string if provided."
                )
1483
1484
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1485

1486
class EmbeddingCompletionRequest(OpenAIBaseModel):
1487
1488
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
1489
1490
    model: str | None = None
    input: list[int] | list[list[int]] | str | list[str]
1491
    encoding_format: Literal["float", "base64"] = "float"
1492
1493
1494
    dimensions: int | None = None
    user: str | None = None
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1495

1496
    # --8<-- [start:embedding-extra-params]
1497
1498
1499
1500
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
1501
1502
            "the prompt."
        ),
1503
    )
1504
1505
1506
1507
1508
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1509
1510
            "if the served model does not use priority scheduling."
        ),
1511
    )
1512
1513
1514
1515
1516
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1517
1518
            "through out the inference process and return in response."
        ),
1519
    )
1520
    normalize: bool | None = None
1521

1522
    # --8<-- [end:embedding-extra-params]
1523

1524
    def to_pooling_params(self):
1525
1526
1527
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
1528
1529
            normalize=self.normalize,
        )
1530
1531


1532
class EmbeddingChatRequest(OpenAIBaseModel):
1533
    model: str | None = None
1534
    messages: list[ChatCompletionMessageParam]
1535
1536

    encoding_format: Literal["float", "base64"] = "float"
1537
1538
1539
    dimensions: int | None = None
    user: str | None = None
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1540

1541
    # --8<-- [start:chat-embedding-extra-params]
1542
1543
    add_generation_prompt: bool = Field(
        default=False,
1544
1545
1546
1547
1548
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
1549
1550
    )

1551
1552
1553
1554
1555
1556
1557
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
1558
1559
            "default)."
        ),
1560
    )
1561
    chat_template: str | None = Field(
1562
1563
1564
1565
1566
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
1567
1568
            "does not define one."
        ),
1569
    )
1570
    chat_template_kwargs: dict[str, Any] | None = Field(
1571
        default=None,
1572
1573
        description=(
            "Additional keyword args to pass to the template renderer. "
1574
1575
            "Will be accessible by the chat template."
        ),
1576
    )
1577
    mm_processor_kwargs: dict[str, Any] | None = Field(
1578
1579
1580
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
1581
1582
1583
1584
1585
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1586
1587
            "if the served model does not use priority scheduling."
        ),
1588
    )
1589
1590
1591
1592
1593
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1594
1595
            "through out the inference process and return in response."
        ),
1596
    )
1597
    normalize: bool | None = None
1598
    # --8<-- [end:chat-embedding-extra-params]
1599
1600
1601
1602

    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
1603
1604
1605
1606
1607
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
1608
1609
1610
        return data

    def to_pooling_params(self):
1611
1612
1613
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
1614
1615
            normalize=self.normalize,
        )
1616
1617


1618
EmbeddingRequest: TypeAlias = EmbeddingCompletionRequest | EmbeddingChatRequest
1619

1620
1621
PoolingCompletionRequest = EmbeddingCompletionRequest
PoolingChatRequest = EmbeddingChatRequest
1622
1623
1624
1625
1626

T = TypeVar("T")


class IOProcessorRequest(OpenAIBaseModel, Generic[T]):
1627
    model: str | None = None
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639

    priority: int = Field(default=0)
    """
    The priority of the request (lower means earlier handling;
    default: 0). Any priority other than 0 will raise an error
    if the served model does not use priority scheduling.
    """
    data: T
    """
    When using plugins IOProcessor plugins, the actual input is processed
    by the plugin itself. Hence, we use a generic type for the request data
    """
1640
    softmax: bool = True
1641
1642

    def to_pooling_params(self):
1643
        return PoolingParams(task="encode", softmax=self.softmax)
1644
1645
1646


class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
1647
    request_id: str | None = None
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
    """
    The request_id associated with this response
    """
    created_at: int = Field(default_factory=lambda: int(time.time()))

    data: T
    """
    When using plugins IOProcessor plugins, the actual output is generated
    by the plugin itself. Hence, we use a generic type for the response data
    """


1660
1661
1662
PoolingRequest: TypeAlias = (
    PoolingCompletionRequest | PoolingChatRequest | IOProcessorRequest
)
1663

1664

1665
class ScoreRequest(OpenAIBaseModel):
1666
1667
1668
1669
    model: str | None = None
    text_1: list[str] | str | ScoreMultiModalParam
    text_2: list[str] | str | ScoreMultiModalParam
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1670

1671
    # --8<-- [start:score-extra-params]
1672

1673
    mm_processor_kwargs: dict[str, Any] | None = Field(
1674
1675
1676
1677
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )

1678
1679
1680
1681
1682
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1683
1684
            "if the served model does not use priority scheduling."
        ),
1685
    )
1686

1687
    activation: bool | None = None
1688

1689
    # --8<-- [end:score-extra-params]
1690

1691
    def to_pooling_params(self):
1692
1693
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1694
1695
            activation=self.activation,
        )
1696
1697


1698
class RerankRequest(OpenAIBaseModel):
1699
1700
1701
    model: str | None = None
    query: str | ScoreMultiModalParam
    documents: list[str] | ScoreMultiModalParam
1702
    top_n: int = Field(default_factory=lambda: 0)
1703
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1704

1705
    # --8<-- [start:rerank-extra-params]
1706

1707
    mm_processor_kwargs: dict[str, Any] | None = Field(
1708
1709
1710
1711
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )

1712
1713
1714
1715
1716
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1717
1718
            "if the served model does not use priority scheduling."
        ),
1719
    )
1720

1721
    activation: bool | None = None
1722

1723
    # --8<-- [end:rerank-extra-params]
1724

1725
    def to_pooling_params(self):
1726
1727
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1728
1729
            activation=self.activation,
        )
1730
1731
1732


class RerankDocument(BaseModel):
1733
1734
    text: str | None = None
    multi_modal: ScoreContentPartParam | None = None
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750


class RerankResult(BaseModel):
    index: int
    document: RerankDocument
    relevance_score: float


class RerankUsage(BaseModel):
    total_tokens: int


class RerankResponse(OpenAIBaseModel):
    id: str
    model: str
    usage: RerankUsage
1751
    results: list[RerankResult]
1752
1753


1754
class CompletionLogProbs(OpenAIBaseModel):
1755
    text_offset: list[int] = Field(default_factory=list)
1756
    token_logprobs: list[float | None] = Field(default_factory=list)
1757
    tokens: list[str] = Field(default_factory=list)
1758
    top_logprobs: list[dict[str, float] | None] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
1759
1760


1761
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1762
1763
    index: int
    text: str
1764
1765
1766
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1767
1768
1769
1770
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1771
1772
            "including encountering the EOS token"
        ),
1773
    )
1774
1775
1776
    token_ids: list[int] | None = None  # For response
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None  # For prompt
Zhuohan Li's avatar
Zhuohan Li committed
1777
1778


1779
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1780
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
1781
    object: Literal["text_completion"] = "text_completion"
Zhuohan Li's avatar
Zhuohan Li committed
1782
1783
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1784
    choices: list[CompletionResponseChoice]
1785
1786
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1787
    usage: UsageInfo
1788
1789

    # vLLM-specific fields that are not in OpenAI spec
1790
    kv_transfer_params: dict[str, Any] | None = Field(
1791
1792
        default=None, description="KVTransfer parameters."
    )
Zhuohan Li's avatar
Zhuohan Li committed
1793
1794


1795
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1796
1797
    index: int
    text: str
1798
1799
1800
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1801
1802
1803
1804
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1805
1806
            "including encountering the EOS token"
        ),
1807
    )
1808
1809
    # not part of the OpenAI spec but for tracing the tokens
    # prompt tokens is put into choice to align with CompletionResponseChoice
1810
1811
    prompt_token_ids: list[int] | None = None
    token_ids: list[int] | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1812
1813


1814
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1815
1816
1817
1818
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1819
    choices: list[CompletionResponseStreamChoice]
1820
    usage: UsageInfo | None = Field(default=None)
1821
1822


1823
class EmbeddingResponseData(OpenAIBaseModel):
1824
1825
    index: int
    object: str = "embedding"
1826
    embedding: list[float] | str
1827
1828


1829
class EmbeddingResponse(OpenAIBaseModel):
1830
    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
1831
1832
1833
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1834
    data: list[EmbeddingResponseData]
1835
1836
1837
    usage: UsageInfo


1838
1839
1840
class PoolingResponseData(OpenAIBaseModel):
    index: int
    object: str = "pooling"
1841
    data: list[list[float]] | list[float] | str
1842
1843
1844
1845
1846
1847
1848


class PoolingResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"pool-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1849
    data: list[PoolingResponseData]
1850
1851
1852
    usage: UsageInfo


1853
1854
1855
class ScoreResponseData(OpenAIBaseModel):
    index: int
    object: str = "score"
1856
    score: float
1857
1858
1859
1860
1861
1862
1863


class ScoreResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1864
    data: list[ScoreResponseData]
1865
1866
1867
    usage: UsageInfo


1868
class ClassificationRequest(OpenAIBaseModel):
1869
1870
1871
1872
    model: str | None = None
    input: list[str] | str
    truncate_prompt_tokens: int | None = None
    user: str | None = None
1873

1874
    # --8<-- [start:classification-extra-params]
1875
1876
1877
1878
1879
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1880
1881
            "if the served model does not use priority scheduling."
        ),
1882
1883
    )

1884
    activation: bool | None = None
1885

1886
    # --8<-- [end:classification-extra-params]
1887
1888

    def to_pooling_params(self):
1889
1890
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1891
1892
            activation=self.activation,
        )
1893
1894
1895
1896


class ClassificationData(OpenAIBaseModel):
    index: int
1897
    label: str | None
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
    probs: list[float]
    num_classes: int


class ClassificationResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"classify-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    data: list[ClassificationData]
    usage: UsageInfo


1911
1912
1913
1914
1915
1916
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
1917
    id: str = Field(default_factory=make_tool_call_id)
1918
1919
1920
1921
    type: Literal["function"] = "function"
    function: FunctionCall


1922
class DeltaFunctionCall(BaseModel):
1923
1924
    name: str | None = None
    arguments: str | None = None
1925
1926
1927
1928


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
1929
1930
    id: str | None = None
    type: Literal["function"] | None = None
1931
    index: int
1932
    function: DeltaFunctionCall | None = None
1933
1934
1935
1936
1937
1938
1939


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
1940
    tool_calls: list[ToolCall]
1941
1942
1943

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
1944
    content: str | None = None
1945
1946


1947
class ChatMessage(OpenAIBaseModel):
1948
    role: str
1949
1950
1951
1952
1953
    content: str | None = None
    refusal: str | None = None
    annotations: OpenAIAnnotation | None = None
    audio: OpenAIChatCompletionAudio | None = None
    function_call: FunctionCall | None = None
1954
    tool_calls: list[ToolCall] = Field(default_factory=list)
1955

1956
    # vLLM-specific fields that are not in OpenAI spec
1957
    reasoning_content: str | None = None
1958

1959

1960
1961
1962
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
1963
    bytes: list[int] | None = None
1964
1965
1966


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
1967
1968
    # Workaround: redefine fields name cache so that it's not
    # shared with the super class.
1969
    field_names: ClassVar[set[str] | None] = None
1970
    top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
1971
1972
1973


class ChatCompletionLogProbs(OpenAIBaseModel):
1974
    content: list[ChatCompletionLogProbsContent] | None = None
1975
1976


1977
class ChatCompletionResponseChoice(OpenAIBaseModel):
1978
1979
    index: int
    message: ChatMessage
1980
    logprobs: ChatCompletionLogProbs | None = None
1981
    # per OpenAI spec this is the default
1982
    finish_reason: str | None = "stop"
1983
    # not part of the OpenAI spec but included in vLLM for legacy reasons
1984
    stop_reason: int | str | None = None
1985
1986
    # not part of the OpenAI spec but is useful for tracing the tokens
    # in agent scenarios
1987
    token_ids: list[int] | None = None
1988
1989


1990
class ChatCompletionResponse(OpenAIBaseModel):
1991
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
1992
    object: Literal["chat.completion"] = "chat.completion"
1993
1994
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1995
    choices: list[ChatCompletionResponseChoice]
1996
1997
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
1998
    usage: UsageInfo
1999
2000

    # vLLM-specific fields that are not in OpenAI spec
2001
2002
2003
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None
    kv_transfer_params: dict[str, Any] | None = Field(
2004
2005
        default=None, description="KVTransfer parameters."
    )
2006
2007


2008
class DeltaMessage(OpenAIBaseModel):
2009
2010
2011
    role: str | None = None
    content: str | None = None
    reasoning_content: str | None = None
2012
    tool_calls: list[DeltaToolCall] = Field(default_factory=list)
2013
2014


2015
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
2016
2017
    index: int
    delta: DeltaMessage
2018
2019
2020
    logprobs: ChatCompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2021
    # not part of the OpenAI spec but for tracing the tokens
2022
    token_ids: list[int] | None = None
2023
2024


2025
class ChatCompletionStreamResponse(OpenAIBaseModel):
2026
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
2027
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
2028
2029
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
2030
    choices: list[ChatCompletionResponseStreamChoice]
2031
    usage: UsageInfo | None = Field(default=None)
2032
    # not part of the OpenAI spec but for tracing the tokens
2033
    prompt_token_ids: list[int] | None = None
2034
2035


2036
2037
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
2038
2039
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2040
2041
2042
2043
2044
2045
2046
2047


class TranscriptionStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
    object: Literal["transcription.chunk"] = "transcription.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranscriptionResponseStreamChoice]
2048
    usage: UsageInfo | None = Field(default=None)
2049
2050


2051
2052
2053
2054
2055
class InputTokensDetails(OpenAIBaseModel):
    cached_tokens: int


class OutputTokensDetails(OpenAIBaseModel):
2056
2057
    reasoning_tokens: int = 0
    tool_output_tokens: int = 0
2058
2059
2060
2061
2062
2063
2064
2065


class ResponseUsage(OpenAIBaseModel):
    input_tokens: int
    input_tokens_details: InputTokensDetails
    output_tokens: int
    output_tokens_details: OutputTokensDetails
    total_tokens: int
2066
2067


2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
def serialize_message(msg):
    """
    Serializes a single message
    """
    if isinstance(msg, dict):
        return msg
    elif hasattr(msg, "__dict__"):
        return msg.to_dict()
    else:
        # fallback to pyandic dump
        return msg.model_dump_json()


def serialize_messages(msgs):
    """
    Serializes multiple messages
    """
    return [serialize_message(msg) for msg in msgs] if msgs else None


2088
2089
2090
2091
class ResponsesResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
    created_at: int = Field(default_factory=lambda: int(time.time()))
    # error: Optional[ResponseError] = None
2092
2093
2094
    incomplete_details: IncompleteDetails | None = None
    instructions: str | None = None
    metadata: Metadata | None = None
2095
2096
    model: str
    object: Literal["response"] = "response"
2097
    output: list[ResponseOutputItem]
2098
2099
2100
2101
2102
2103
2104
    parallel_tool_calls: bool
    temperature: float
    tool_choice: ToolChoice
    tools: list[Tool]
    top_p: float
    background: bool
    max_output_tokens: int
2105
2106
2107
2108
    max_tool_calls: int | None = None
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
2109
2110
    service_tier: Literal["auto", "default", "flex", "scale", "priority"]
    status: ResponseStatus
2111
2112
    text: ResponseTextConfig | None = None
    top_logprobs: int | None = None
2113
    truncation: Literal["auto", "disabled"]
2114
2115
    usage: ResponseUsage | None = None
    user: str | None = None
2116

2117
2118
2119
2120
    # --8<-- [start:responses-extra-params]
    # These are populated when enable_response_messages is set to True
    # NOTE: custom serialization is needed
    # see serialize_input_messages and serialize_output_messages
2121
2122
    input_messages: list[ChatCompletionMessageParam] | None = None
    output_messages: list[ChatCompletionMessageParam] | None = None
2123
2124
2125
2126
2127
2128
2129
    # --8<-- [end:responses-extra-params]

    # NOTE: openAI harmony doesn't serialize TextContent properly,
    # TODO: this fixes for TextContent, but need to verify for tools etc
    # https://github.com/openai/harmony/issues/78
    @field_serializer("output_messages", when_used="json")
    def serialize_output_messages(self, msgs, _info):
2130
        return serialize_messages(msgs)
2131
2132
2133
2134
2135

    # NOTE: openAI harmony doesn't serialize TextContent properly, this fixes it
    # https://github.com/openai/harmony/issues/78
    @field_serializer("input_messages", when_used="json")
    def serialize_input_messages(self, msgs, _info):
2136
        return serialize_messages(msgs)
2137

2138
2139
2140
2141
2142
2143
2144
2145
2146
    @classmethod
    def from_request(
        cls,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
        model_name: str,
        created_time: int,
        output: list[ResponseOutputItem],
        status: ResponseStatus,
2147
2148
2149
        usage: ResponseUsage | None = None,
        input_messages: list[ChatCompletionMessageParam] | None = None,
        output_messages: list[ChatCompletionMessageParam] | None = None,
2150
    ) -> "ResponsesResponse":
2151
        incomplete_details: IncompleteDetails | None = None
2152
2153
        if status == "incomplete":
            incomplete_details = IncompleteDetails(reason="max_output_tokens")
2154
2155
2156
        # TODO: implement the other reason for incomplete_details,
        # which is content_filter
        # incomplete_details = IncompleteDetails(reason='content_filter')
2157
2158
2159
        return cls(
            id=request.request_id,
            created_at=created_time,
2160
            incomplete_details=incomplete_details,
2161
2162
2163
2164
            instructions=request.instructions,
            metadata=request.metadata,
            model=model_name,
            output=output,
2165
2166
            input_messages=input_messages,
            output_messages=output_messages,
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
            parallel_tool_calls=request.parallel_tool_calls,
            temperature=sampling_params.temperature,
            tool_choice=request.tool_choice,
            tools=request.tools,
            top_p=sampling_params.top_p,
            background=request.background,
            max_output_tokens=sampling_params.max_tokens,
            max_tool_calls=request.max_tool_calls,
            previous_response_id=request.previous_response_id,
            prompt=request.prompt,
            reasoning=request.reasoning,
            service_tier=request.service_tier,
            status=status,
            text=request.text,
            top_logprobs=sampling_params.logprobs,
            truncation=request.truncation,
            user=request.user,
            usage=usage,
        )


2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartDoneEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.done"]
    """The type of the event. Always `response.reasoning_part.done`."""


# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartAddedEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.added"]
    """The type of the event. Always `response.reasoning_part.added`."""


2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
# vLLM Streaming Events
# Note: we override the response type with the vLLM ResponsesResponse type
class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseInProgressEvent(OpenAIResponseInProgressEvent):
    response: ResponsesResponse  # type: ignore[override]


2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
StreamingResponsesResponse: TypeAlias = (
    ResponseCreatedEvent
    | ResponseInProgressEvent
    | ResponseCompletedEvent
    | ResponseOutputItemAddedEvent
    | ResponseOutputItemDoneEvent
    | ResponseContentPartAddedEvent
    | ResponseContentPartDoneEvent
    | ResponseReasoningTextDeltaEvent
    | ResponseReasoningTextDoneEvent
    | ResponseReasoningPartAddedEvent
    | ResponseReasoningPartDoneEvent
    | ResponseCodeInterpreterCallInProgressEvent
    | ResponseCodeInterpreterCallCodeDeltaEvent
    | ResponseWebSearchCallInProgressEvent
    | ResponseWebSearchCallSearchingEvent
    | ResponseWebSearchCallCompletedEvent
    | ResponseCodeInterpreterCallCodeDoneEvent
    | ResponseCodeInterpreterCallInterpretingEvent
    | ResponseCodeInterpreterCallCompletedEvent
)
2267

2268
2269
2270
BatchRequestInputBody: TypeAlias = (
    ChatCompletionRequest | EmbeddingRequest | ScoreRequest | RerankRequest
)
2271
2272


2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

2292
    # The parameters of the request.
2293
    body: BatchRequestInputBody
2294

2295
    @field_validator("body", mode="plain")
2296
2297
2298
    @classmethod
    def check_type_for_url(cls, value: Any, info: ValidationInfo):
        # Use url to disambiguate models
2299
        url: str = info.data["url"]
2300
2301
2302
2303
        if url == "/v1/chat/completions":
            return ChatCompletionRequest.model_validate(value)
        if url == "/v1/embeddings":
            return TypeAdapter(EmbeddingRequest).validate_python(value)
2304
        if url.endswith("/score"):
2305
            return ScoreRequest.model_validate(value)
2306
2307
2308
        if url.endswith("/rerank"):
            return RerankRequest.model_validate(value)
        return TypeAdapter(BatchRequestInputBody).validate_python(value)
2309

2310

2311
2312
2313
2314
2315
2316
2317
2318
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
2319
2320
2321
2322
2323
2324
2325
    body: (
        ChatCompletionResponse
        | EmbeddingResponse
        | ScoreResponse
        | RerankResponse
        | None
    ) = None
2326
2327


2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

2339
    response: BatchResponseData | None
2340
2341
2342

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
2343
    error: Any | None
2344
2345


2346
class TokenizeCompletionRequest(OpenAIBaseModel):
2347
    model: str | None = None
2348
2349
    prompt: str

2350
2351
2352
2353
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
2354
2355
            "the prompt."
        ),
2356
    )
2357
    return_token_strs: bool | None = Field(
2358
        default=False,
2359
2360
2361
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
2362
    )
2363
2364
2365


class TokenizeChatRequest(OpenAIBaseModel):
2366
    model: str | None = None
2367
    messages: list[ChatCompletionMessageParam]
2368

2369
2370
    add_generation_prompt: bool = Field(
        default=True,
2371
2372
2373
2374
2375
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
2376
    )
2377
    return_token_strs: bool | None = Field(
2378
        default=False,
2379
2380
2381
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
2382
    )
2383
2384
    continue_final_message: bool = Field(
        default=False,
2385
2386
2387
2388
2389
2390
2391
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
2392
2393
2394
2395
2396
2397
2398
2399
    )
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
2400
2401
            "default)."
        ),
2402
    )
2403
    chat_template: str | None = Field(
2404
2405
2406
2407
2408
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
2409
2410
            "does not define one."
        ),
2411
    )
2412
    chat_template_kwargs: dict[str, Any] | None = Field(
2413
        default=None,
2414
2415
        description=(
            "Additional keyword args to pass to the template renderer. "
2416
2417
            "Will be accessible by the chat template."
        ),
2418
    )
2419
    mm_processor_kwargs: dict[str, Any] | None = Field(
2420
2421
2422
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
2423
    tools: list[ChatCompletionToolsParam] | None = Field(
2424
2425
2426
        default=None,
        description=("A list of tools the model may call."),
    )
2427

2428
2429
2430
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
2431
2432
2433
2434
2435
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
2436
2437
        return data

2438

2439
TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest
2440
2441
2442
2443
2444


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
2445
    tokens: list[int]
2446
    token_strs: list[str] | None = None
2447
2448
2449


class DetokenizeRequest(OpenAIBaseModel):
2450
    model: str | None = None
2451
    tokens: list[int]
2452
2453
2454
2455


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
2456
2457


2458
2459
class TokenizerInfoResponse(OpenAIBaseModel):
    """
2460
    Response containing tokenizer configuration
2461
2462
2463
2464
2465
2466
2467
    equivalent to tokenizer_config.json
    """

    model_config = ConfigDict(extra="allow")
    tokenizer_class: str


2468
class LoadLoRAAdapterRequest(BaseModel):
2469
2470
2471
2472
    lora_name: str
    lora_path: str


2473
class UnloadLoRAAdapterRequest(BaseModel):
2474
    lora_name: str
2475
    lora_int_id: int | None = Field(default=None)
2476
2477
2478


## Protocols for Audio
2479
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
2480
2481
2482
2483


class TranscriptionRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
2484
    # https://platform.openai.com/docs/api-reference/audio/createTranscription
2485
2486
2487
2488
2489
2490
2491

    file: UploadFile
    """
    The audio file object (not file name) to transcribe, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

2492
    model: str | None = None
2493
2494
2495
    """ID of the model to use.
    """

2496
    language: str | None = None
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
    """The language of the input audio.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy and latency.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    ## TODO (varun) : Support if set to 0, certain thresholds are met !!

2520
    timestamp_granularities: list[Literal["word", "segment"]] = Field(
2521
2522
        alias="timestamp_granularities[]", default=[]
    )
2523
2524
2525
2526
2527
2528
2529
2530
    """The timestamp granularities to populate for this transcription.

    `response_format` must be set `verbose_json` to use timestamp granularities.
    Either or both of these options are supported: `word`, or `segment`. Note:
    There is no additional latency for segment timestamps, but generating word
    timestamps incurs additional latency.
    """

2531
    stream: bool | None = False
2532
    """When set, it will enable output to be streamed in a similar fashion
2533
    as the Chat Completion endpoint.
2534
    """
2535
    # --8<-- [start:transcription-extra-params]
2536
    # Flattened stream option to simplify form data.
2537
2538
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2539

2540
    vllm_xargs: dict[str, str | int | float] | None = Field(
2541
        default=None,
2542
2543
2544
2545
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
2546
    )
2547
    # --8<-- [end:transcription-extra-params]
2548

2549
    to_language: str | None = None
2550
2551
    """The language of the output audio we transcribe to.

2552
    Please note that this is not currently used by supported models at this
2553
2554
2555
    time, but it is a placeholder for future use, matching translation api.
    """

2556
    # --8<-- [start:transcription-sampling-params]
2557
2558
2559
2560
2561
2562
2563
2564
2565
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """

2566
    top_p: float | None = None
2567
    """Enables nucleus (top-p) sampling, where tokens are selected from the
2568
2569
2570
    smallest possible set whose cumulative probability exceeds `p`.
    """

2571
    top_k: int | None = None
2572
2573
    """Limits sampling to the `k` most probable tokens at each step."""

2574
    min_p: float | None = None
2575
    """Filters out tokens with a probability lower than `min_p`, ensuring a
2576
2577
2578
    minimum likelihood threshold during sampling.
    """

2579
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2580
2581
    """The seed to use for sampling."""

2582
    frequency_penalty: float | None = 0.0
2583
2584
    """The frequency penalty to use for sampling."""

2585
    repetition_penalty: float | None = None
2586
2587
    """The repetition penalty to use for sampling."""

2588
    presence_penalty: float | None = 0.0
2589
    """The presence penalty to use for sampling."""
2590
    # --8<-- [end:transcription-sampling-params]
2591

2592
2593
    # Default sampling parameters for transcription requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
2594
2595
2596
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
2597
        "top_k": 0,
2598
        "min_p": 0.0,
2599
2600
2601
    }

    def to_sampling_params(
2602
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2603
    ) -> SamplingParams:
2604
2605
2606
2607
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
2608

2609
2610
2611
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2612
2613
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2614
2615
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
2616
2617
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
2618
2619
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
2620
2621
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
2622
2623
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
2624
2625
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
2626
2627
2628
2629

        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )

        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=repetition_penalty,
            presence_penalty=self.presence_penalty,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
            extra_args=self.vllm_xargs,
        )
2648
2649
2650

    @model_validator(mode="before")
    @classmethod
2651
2652
2653
2654
2655
2656
2657
    def validate_transcription_request(cls, data):
        if isinstance(data.get("file"), str):
            raise HTTPException(
                status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
                detail="Expected 'file' to be a file-like object, not 'str'.",
            )

2658
2659
2660
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
2661
            raise ValueError("Stream options can only be defined when `stream=True`.")
2662
2663

        return data
2664
2665
2666


# Transcription response objects
2667
2668
2669
2670
2671
class TranscriptionUsageAudio(OpenAIBaseModel):
    type: Literal["duration"] = "duration"
    seconds: int


2672
2673
2674
class TranscriptionResponse(OpenAIBaseModel):
    text: str
    """The transcribed text."""
2675
    usage: TranscriptionUsageAudio
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726


class TranscriptionWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranscriptionSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

    avg_logprob: float
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

    compression_ratio: float
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

    no_speech_prob: float
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

2727
    tokens: list[int]
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
    """Array of token IDs for the text content."""


class TranscriptionResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The transcribed text."""

2741
    segments: list[TranscriptionSegment] | None = None
2742
2743
    """Segments of the transcribed text and their corresponding details."""

2744
    words: list[TranscriptionWord] | None = None
2745
    """Extracted words and their corresponding timestamps."""
2746
2747
2748
2749


class TranslationResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
2750
2751
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2752
2753
2754
2755
2756
2757
2758
2759


class TranslationStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
    object: Literal["translation.chunk"] = "translation.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranslationResponseStreamChoice]
2760
    usage: UsageInfo | None = Field(default=None)
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772


class TranslationRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/audio/createTranslation

    file: UploadFile
    """
    The audio file object (not file name) to translate, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

2773
    model: str | None = None
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
    """ID of the model to use.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    # TODO support additional sampling parameters
    # --8<-- [start:translation-sampling-params]
2793
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2794
2795
    """The seed to use for sampling."""

2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """
    # --8<-- [end:translation-sampling-params]

    # --8<-- [start:translation-extra-params]
2807
    language: str | None = None
2808
2809
2810
2811
2812
2813
2814
    """The language of the input audio we translate from.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy.
    """

2815
    to_language: str | None = None
2816
2817
2818
2819
2820
2821
2822
    """The language of the input audio we translate to.

    Please note that this is not supported by all models, refer to the specific
    model documentation for more details.
    For instance, Whisper only supports `to_language=en`.
    """

2823
    stream: bool | None = False
2824
    """Custom field not present in the original OpenAI definition. When set,
2825
    it will enable output to be streamed in a similar fashion as the Chat
2826
    Completion endpoint.
2827
2828
    """
    # Flattened stream option to simplify form data.
2829
2830
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2831
2832
2833
2834
2835
2836
2837
2838
    # --8<-- [end:translation-extra-params]

    # Default sampling parameters for translation requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "temperature": 0,
    }

    def to_sampling_params(
2839
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2840
    ) -> SamplingParams:
2841
2842
2843
2844
2845
2846
2847
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2848
2849
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2850

2851
2852
2853
2854
2855
2856
2857
2858
        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
        )
2859
2860
2861
2862
2863
2864
2865

    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
2866
            raise ValueError("Stream options can only be defined when `stream=True`.")
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939

        return data


# Translation response objects
class TranslationResponse(OpenAIBaseModel):
    text: str
    """The translated text."""


class TranslationWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranslationSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

    avg_logprob: float
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

    compression_ratio: float
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

    no_speech_prob: float
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

    tokens: list[int]
    """Array of token IDs for the text content."""


class TranslationResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The translated text."""

2940
    segments: list[TranslationSegment] | None = None
2941
2942
    """Segments of the translated text and their corresponding details."""

2943
    words: list[TranslationWord] | None = None
2944
    """Extracted words and their corresponding timestamps."""