protocol.py 108 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import json
Zhuohan Li's avatar
Zhuohan Li committed
7
import time
8
from http import HTTPStatus
9
from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
Zhuohan Li's avatar
Zhuohan Li committed
10

11
import regex as re
12
import torch
13
from fastapi import HTTPException, UploadFile
14
from openai.types.chat.chat_completion_audio import (
15
16
17
    ChatCompletionAudio as OpenAIChatCompletionAudio,
)
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
18
19
20
21
22
from openai.types.responses import (
    ResponseCodeInterpreterCallCodeDeltaEvent,
    ResponseCodeInterpreterCallCodeDoneEvent,
    ResponseCodeInterpreterCallCompletedEvent,
    ResponseCodeInterpreterCallInProgressEvent,
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
    ResponseCodeInterpreterCallInterpretingEvent,
    ResponseContentPartAddedEvent,
    ResponseContentPartDoneEvent,
    ResponseFunctionToolCall,
    ResponseInputItemParam,
    ResponseOutputItem,
    ResponseOutputItemAddedEvent,
    ResponseOutputItemDoneEvent,
    ResponsePrompt,
    ResponseReasoningTextDeltaEvent,
    ResponseReasoningTextDoneEvent,
    ResponseStatus,
    ResponseWebSearchCallCompletedEvent,
    ResponseWebSearchCallInProgressEvent,
    ResponseWebSearchCallSearchingEvent,
)
39
from openai.types.responses import (
40
41
42
    ResponseCompletedEvent as OpenAIResponseCompletedEvent,
)
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
43
from openai.types.responses import (
44
45
    ResponseInProgressEvent as OpenAIResponseInProgressEvent,
)
46
from openai.types.responses.response_reasoning_item import (
47
48
    Content as ResponseReasoningTextContent,
)
49
from openai_harmony import Message as OpenAIHarmonyMessage
50

51
52
from vllm.config.pooler import get_use_activation
from vllm.tasks import PoolingTask
53
54
55
56
57
58
from vllm.utils.serial_utils import (
    EmbedDType,
    EncodingFormat,
    Endianness,
)

59
60
61
62
# Backward compatibility for OpenAI client versions
try:  # For older openai versions (< 1.100.0)
    from openai.types.responses import ResponseTextConfig
except ImportError:  # For newer openai versions (>= 1.100.0)
63
    from openai.types.responses import ResponseFormatTextConfig as ResponseTextConfig
64

65

66
from openai.types.responses.response import IncompleteDetails, ToolChoice
67
68
from openai.types.responses.tool import Tool
from openai.types.shared import Metadata, Reasoning
69
70
71
72
73
from pydantic import (
    BaseModel,
    ConfigDict,
    Field,
    TypeAdapter,
74
    ValidationError,
75
    ValidationInfo,
76
    field_serializer,
77
78
79
    field_validator,
    model_validator,
)
Zhuohan Li's avatar
Zhuohan Li committed
80

81
82
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam, make_tool_call_id
from vllm.entrypoints.score_utils import ScoreContentPartParam, ScoreMultiModalParam
83
from vllm.logger import init_logger
84
from vllm.logprobs import Logprob
85
from vllm.pooling_params import PoolingParams
86
87
88
89
90
91
from vllm.sampling_params import (
    BeamSearchParams,
    RequestOutputKind,
    SamplingParams,
    StructuredOutputsParams,
)
92
93
from vllm.utils import random_uuid
from vllm.utils.import_utils import resolve_obj_by_qualname
94

95
96
logger = init_logger(__name__)

97
_LONG_INFO = torch.iinfo(torch.long)
98

Zhuohan Li's avatar
Zhuohan Li committed
99

100
class OpenAIBaseModel(BaseModel):
101
102
103
    # OpenAI API does allow extra fields
    model_config = ConfigDict(extra="allow")

104
    # Cache class field names
105
    field_names: ClassVar[set[str] | None] = None
106

107
    @model_validator(mode="wrap")
108
    @classmethod
109
110
111
112
    def __log_extra_fields__(cls, data, handler):
        result = handler(data)
        if not isinstance(data, dict):
            return result
113
114
        field_names = cls.field_names
        if field_names is None:
115
116
117
118
            # Get all class field names and their potential aliases
            field_names = set()
            for field_name, field in cls.model_fields.items():
                field_names.add(field_name)
119
                if alias := getattr(field, "alias", None):
120
121
122
123
124
125
                    field_names.add(alias)
            cls.field_names = field_names

        # Compare against both field names and aliases
        if any(k not in field_names for k in data):
            logger.warning(
126
                "The following fields were present in the request but ignored: %s",
127
128
                data.keys() - field_names,
            )
129
        return result
130
131


132
class ErrorInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
133
134
    message: str
    type: str
135
    param: str | None = None
136
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
137
138


139
140
141
142
class ErrorResponse(OpenAIBaseModel):
    error: ErrorInfo


143
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
144
145
146
147
148
149
150
151
152
153
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
154
    group: str | None = None
155
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
156
157


158
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
159
160
161
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
162
    owned_by: str = "vllm"
163
164
165
    root: str | None = None
    parent: str | None = None
    max_model_len: int | None = None
166
    permission: list[ModelPermission] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
167
168


169
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
170
    object: str = "list"
171
    data: list[ModelCard] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
172
173


174
class PromptTokenUsageInfo(OpenAIBaseModel):
175
    cached_tokens: int | None = None
176
177


178
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
179
180
    prompt_tokens: int = 0
    total_tokens: int = 0
181
182
    completion_tokens: int | None = 0
    prompt_tokens_details: PromptTokenUsageInfo | None = None
Zhuohan Li's avatar
Zhuohan Li committed
183
184


185
186
class RequestResponseMetadata(BaseModel):
    request_id: str
187
    final_usage_info: UsageInfo | None = None
188
189


190
191
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
192
    description: str | None = None
193
194
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
195
196
    json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
    strict: bool | None = None
197
198


199
class LegacyStructuralTag(OpenAIBaseModel):
200
201
202
    begin: str
    # schema is the field, but that causes conflicts with pydantic so
    # instead use structural_tag_schema with an alias
203
    structural_tag_schema: dict[str, Any] | None = Field(default=None, alias="schema")
204
205
206
    end: str


207
class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
208
    type: Literal["structural_tag"]
209
    structures: list[LegacyStructuralTag]
210
211
212
    triggers: list[str]


213
214
215
216
217
218
219
220
221
222
class StructuralTagResponseFormat(OpenAIBaseModel):
    type: Literal["structural_tag"]
    format: Any


AnyStructuralTagResponseFormat: TypeAlias = (
    LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
)


223
class ResponseFormat(OpenAIBaseModel):
224
    # type must be "json_schema", "json_object", or "text"
225
    type: Literal["text", "json_object", "json_schema"]
226
    json_schema: JsonSchemaResponseFormat | None = None
227
228


229
230
231
AnyResponseFormat: TypeAlias = (
    ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
)
232
233


234
class StreamOptions(OpenAIBaseModel):
235
236
    include_usage: bool | None = True
    continuous_usage_stats: bool | None = False
237
238


239
240
class FunctionDefinition(OpenAIBaseModel):
    name: str
241
242
    description: str | None = None
    parameters: dict[str, Any] | None = None
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


259
260
# extra="forbid" is a workaround to have kwargs as a field,
# see https://github.com/pydantic/pydantic/issues/3125
261
262
class LogitsProcessorConstructor(BaseModel):
    qualname: str
263
264
    args: list[Any] | None = None
    kwargs: dict[str, Any] | None = None
265

266
267
    model_config = ConfigDict(extra="forbid")

268

269
LogitsProcessors = list[str | LogitsProcessorConstructor]
270
271


272
def get_logits_processors(
273
274
    processors: LogitsProcessors | None, pattern: str | None
) -> list[Any] | None:
275
276
277
    if processors and pattern:
        logits_processors = []
        for processor in processors:
278
            qualname = processor if isinstance(processor, str) else processor.qualname
279
280
281
282
            if not re.match(pattern, qualname):
                raise ValueError(
                    f"Logits processor '{qualname}' is not allowed by this "
                    "server. See --logits-processor-pattern engine argument "
283
284
                    "for more information."
                )
285
286
287
288
289
290
291
            try:
                logits_processor = resolve_obj_by_qualname(qualname)
            except Exception as e:
                raise ValueError(
                    f"Logits processor '{qualname}' could not be resolved: {e}"
                ) from e
            if isinstance(processor, LogitsProcessorConstructor):
292
293
294
                logits_processor = logits_processor(
                    *processor.args or [], **processor.kwargs or {}
                )
295
296
297
298
299
            logits_processors.append(logits_processor)
        return logits_processors
    elif processors:
        raise ValueError(
            "The `logits_processors` argument is not supported by this "
300
            "server. See --logits-processor-pattern engine argument "
301
302
            "for more information."
        )
303
304
305
    return None


306
ResponseInputOutputItem: TypeAlias = ResponseInputItemParam | ResponseOutputItem
307
308


309
310
311
class ResponsesRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/responses/create
312
313
    background: bool | None = False
    include: (
314
315
316
317
318
319
320
321
322
323
        list[
            Literal[
                "code_interpreter_call.outputs",
                "computer_call_output.output.image_url",
                "file_search_call.results",
                "message.input_image.image_url",
                "message.output_text.logprobs",
                "reasoning.encrypted_content",
            ],
        ]
324
325
326
327
328
329
330
331
332
333
334
335
        | None
    ) = None
    input: str | list[ResponseInputOutputItem]
    instructions: str | None = None
    max_output_tokens: int | None = None
    max_tool_calls: int | None = None
    metadata: Metadata | None = None
    model: str | None = None
    parallel_tool_calls: bool | None = True
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
336
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto"
337
338
339
340
    store: bool | None = True
    stream: bool | None = False
    temperature: float | None = None
    text: ResponseTextConfig | None = None
341
342
    tool_choice: ToolChoice = "auto"
    tools: list[Tool] = Field(default_factory=list)
343
344
345
346
    top_logprobs: int | None = 0
    top_p: float | None = None
    truncation: Literal["auto", "disabled"] | None = "disabled"
    user: str | None = None
347
348
349
350
351
352
353

    # --8<-- [start:responses-extra-params]
    request_id: str = Field(
        default_factory=lambda: f"resp_{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
354
355
            "through out the inference process and return in response."
        ),
356
    )
357
    mm_processor_kwargs: dict[str, Any] | None = Field(
358
359
360
361
362
363
364
365
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
366
367
            "if the served model does not use priority scheduling."
        ),
368
    )
369
    cache_salt: str | None = Field(
370
371
372
373
374
375
376
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
377
            "to 256 bit)."
378
379
        ),
    )
380
381
382
383
384

    enable_response_messages: bool = Field(
        default=False,
        description=(
            "Dictates whether or not to return messages as part of the "
385
            "response object. Currently only supported for"
386
387
388
            "non-background and gpt-oss only. "
        ),
    )
389
390
391
392
393
    # similar to input_messages / output_messages in ResponsesResponse
    # we take in previous_input_messages (ie in harmony format)
    # this cannot be used in conjunction with previous_response_id
    # TODO: consider supporting non harmony messages as well
    previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None
394
395
396
397
398
399
400
401
402
403
    # --8<-- [end:responses-extra-params]

    _DEFAULT_SAMPLING_PARAMS = {
        "temperature": 1.0,
        "top_p": 1.0,
    }

    def to_sampling_params(
        self,
        default_max_tokens: int,
404
        default_sampling_params: dict | None = None,
405
406
407
408
409
410
411
412
413
    ) -> SamplingParams:
        if self.max_output_tokens is None:
            max_tokens = default_max_tokens
        else:
            max_tokens = min(self.max_output_tokens, default_max_tokens)

        default_sampling_params = default_sampling_params or {}
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
414
415
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
416
417
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
418
419
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
420
        stop_token_ids = default_sampling_params.get("stop_token_ids")
421
422

        # Structured output
423
        structured_outputs = None
424
425
        if self.text is not None and self.text.format is not None:
            response_format = self.text.format
426
427
428
429
            if (
                response_format.type == "json_schema"
                and response_format.schema_ is not None
            ):
430
                structured_outputs = StructuredOutputsParams(
431
432
                    json=response_format.schema_
                )
433
434
435
436
437
438
439
440
            elif response_format.type == "json_object":
                raise NotImplementedError("json_object is not supported")

        # TODO: add more parameters
        return SamplingParams.from_optional(
            temperature=temperature,
            top_p=top_p,
            max_tokens=max_tokens,
441
            logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
442
            stop_token_ids=stop_token_ids,
443
444
445
            output_kind=(
                RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
            ),
446
            structured_outputs=structured_outputs,
447
448
        )

449
450
451
452
    def is_include_output_logprobs(self) -> bool:
        """Check if the request includes output logprobs."""
        if self.include is None:
            return False
453
454
455
456
        return (
            isinstance(self.include, list)
            and "message.output_text.logprobs" in self.include
        )
457

458
459
460
461
462
    @model_validator(mode="before")
    def validate_background(cls, data):
        if not data.get("background"):
            return data
        if not data.get("store", True):
463
            raise ValueError("background can only be used when `store` is true")
464
465
466
467
468
469
470
471
        return data

    @model_validator(mode="before")
    def validate_prompt(cls, data):
        if data.get("prompt") is not None:
            raise ValueError("prompt template is not supported")
        return data

472
473
    @model_validator(mode="before")
    def check_cache_salt_support(cls, data):
474
475
476
477
478
479
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
480
481
        return data

482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
    @model_validator(mode="before")
    def function_call_parsing(cls, data):
        """Parse function_call dictionaries into ResponseFunctionToolCall objects.
        This ensures Pydantic can properly resolve union types in the input field.
        Function calls provided as dicts are converted to ResponseFunctionToolCall
        objects before validation, while invalid structures are left for Pydantic
        to reject with appropriate error messages.
        """

        input_data = data.get("input")

        # Early return for None, strings, or bytes
        # (strings are iterable but shouldn't be processed)
        if input_data is None or isinstance(input_data, (str, bytes)):
            return data

        # Convert iterators (like ValidatorIterator) to list
        if not isinstance(input_data, list):
            try:
                input_data = list(input_data)
            except TypeError:
                # Not iterable, leave as-is for Pydantic to handle
                return data

        processed_input = []
        for item in input_data:
            if isinstance(item, dict) and item.get("type") == "function_call":
                try:
                    processed_input.append(ResponseFunctionToolCall(**item))
                except ValidationError:
                    # Let Pydantic handle validation for malformed function calls
                    logger.debug(
                        "Failed to parse function_call to ResponseFunctionToolCall, "
                        "leaving for Pydantic validation"
                    )
                    processed_input.append(item)
            else:
                processed_input.append(item)

        data["input"] = processed_input
        return data

524

525
class ChatCompletionRequest(OpenAIBaseModel):
526
527
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
528
    messages: list[ChatCompletionMessageParam]
529
530
531
532
533
534
    model: str | None = None
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: bool | None = False
    top_logprobs: int | None = 0
    max_tokens: int | None = Field(
535
        default=None,
536
537
        deprecated="max_tokens is deprecated in favor of "
        "the max_completion_tokens field",
538
    )
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
    max_completion_tokens: int | None = None
    n: int | None = 1
    presence_penalty: float | None = 0.0
    response_format: AnyResponseFormat | None = None
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    temperature: float | None = None
    top_p: float | None = None
    tools: list[ChatCompletionToolsParam] | None = None
    tool_choice: (
        Literal["none"]
        | Literal["auto"]
        | Literal["required"]
        | ChatCompletionNamedToolChoiceParam
        | None
    ) = "none"
    reasoning_effort: Literal["low", "medium", "high"] | None = None
558
    include_reasoning: bool = True
559
    parallel_tool_calls: bool | None = True
560

561
    # NOTE this will be ignored by vLLM
562
    user: str | None = None
563

564
    # --8<-- [start:chat-completion-sampling-params]
565
    use_beam_search: bool = False
566
567
568
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
569
    length_penalty: float = 1.0
570
    stop_token_ids: list[int] | None = []
571
572
573
574
575
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
576
577
578
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    prompt_logprobs: int | None = None
    allowed_token_ids: list[int] | None = None
579
    bad_words: list[str] = Field(default_factory=list)
580
    # --8<-- [end:chat-completion-sampling-params]
581

582
    # --8<-- [start:chat-completion-extra-params]
583
    echo: bool = Field(
584
585
586
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
587
588
            "if they belong to the same role."
        ),
589
    )
590
    add_generation_prompt: bool = Field(
591
        default=True,
592
593
594
595
596
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
597
    )
598
599
    continue_final_message: bool = Field(
        default=False,
600
601
602
603
604
605
606
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
607
    )
608
    add_special_tokens: bool = Field(
609
610
611
612
613
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
614
            "special tokens so this should be set to false (as is the "
615
616
            "default)."
        ),
617
    )
618
    documents: list[dict[str, str]] | None = Field(
619
        default=None,
620
621
622
623
624
625
626
        description=(
            "A list of dicts representing documents that will be accessible to "
            "the model if it is performing RAG (retrieval-augmented generation)."
            " If the template does not support RAG, this argument will have no "
            "effect. We recommend that each document should be a dict containing "
            '"title" and "text" keys.'
        ),
627
    )
628
    chat_template: str | None = Field(
629
630
631
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
632
633
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
634
635
            "does not define one."
        ),
636
    )
637
    chat_template_kwargs: dict[str, Any] | None = Field(
638
        default=None,
639
640
        description=(
            "Additional keyword args to pass to the template renderer. "
641
642
            "Will be accessible by the chat template."
        ),
643
    )
644
    mm_processor_kwargs: dict[str, Any] | None = Field(
645
646
647
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
648
    structured_outputs: StructuredOutputsParams | None = Field(
649
        default=None,
650
        description="Additional kwargs for structured outputs",
651
    )
652
653
654
655
656
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
657
658
            "if the served model does not use priority scheduling."
        ),
659
    )
660
    request_id: str = Field(
661
        default_factory=random_uuid,
662
663
664
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
665
666
            "through out the inference process and return in response."
        ),
667
    )
668
    logits_processors: LogitsProcessors | None = Field(
669
670
671
672
673
674
675
676
677
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
678
679
680
            "{'param': 'value'}}."
        ),
    )
681
    return_tokens_as_token_ids: bool | None = Field(
682
683
684
685
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
686
687
688
            "that are not JSON-encodable can be identified."
        ),
    )
689
    return_token_ids: bool | None = Field(
690
691
692
693
694
695
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
696
697
698
            "need to map generated text back to input tokens."
        ),
    )
699
    cache_salt: str | None = Field(
700
701
702
703
704
705
706
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
707
            "to 256 bit)."
708
709
        ),
    )
710
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
711
        default=None,
712
713
        description="KVTransfer parameters used for disaggregated serving.",
    )
714

715
    vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
716
        default=None,
717
        description=(
718
            "Additional request parameters with (list of) string or "
719
720
            "numeric values, used by custom extensions."
        ),
721
722
    )

723
    # --8<-- [end:chat-completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
724

725
726
727
728
729
    # Default sampling parameters for chat completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
730
        "top_k": 0,
731
732
733
734
        "min_p": 0.0,
    }

    def to_beam_search_params(
735
736
        self, max_tokens: int, default_sampling_params: dict
    ) -> BeamSearchParams:
737
        n = self.n if self.n is not None else 1
738
739
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
740
741
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
742
743
744
745
746
747

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
748
            length_penalty=self.length_penalty,
749
750
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
751

752
    def to_sampling_params(
753
        self,
754
        max_tokens: int,
755
        logits_processor_pattern: str | None,
756
        default_sampling_params: dict,
757
    ) -> SamplingParams:
758
759
760
761
762
763
764
765
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
766
767
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
768
769
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
770
771
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
772
773
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
774
775
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
776
777
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
778
779
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
780

781
782
783
784
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

785
        response_format = self.response_format
786
        if response_format is not None:
787
788
789
790
791
792
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
            if response_format.type == "json_object":
                self.structured_outputs.json_object = True
            elif response_format.type == "json_schema":
                json_schema = response_format.json_schema
                assert json_schema is not None
                self.structured_outputs.json = json_schema.json_schema
            elif response_format.type == "structural_tag":
                structural_tag = response_format
                assert structural_tag is not None and isinstance(
                    structural_tag,
                    (
                        LegacyStructuralTagResponseFormat,
                        StructuralTagResponseFormat,
                    ),
                )
                s_tag_obj = structural_tag.model_dump(by_alias=True)
                self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
810

811
812
813
814
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
815
        return SamplingParams.from_optional(
816
817
818
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
819
820
821
822
823
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
824
            seed=self.seed,
825
826
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
827
            logprobs=self.top_logprobs if self.logprobs else None,
828
            prompt_logprobs=prompt_logprobs,
829
            ignore_eos=self.ignore_eos,
830
            max_tokens=max_tokens,
831
            min_tokens=self.min_tokens,
832
833
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
834
835
836
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
837
            include_stop_str_in_output=self.include_stop_str_in_output,
838
            truncate_prompt_tokens=self.truncate_prompt_tokens,
839
840
841
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
842
            structured_outputs=self.structured_outputs,
Robert Shaw's avatar
Robert Shaw committed
843
            logit_bias=self.logit_bias,
844
            bad_words=self.bad_words,
845
            allowed_token_ids=self.allowed_token_ids,
846
847
            extra_args=extra_args or None,
        )
848

849
    @model_validator(mode="before")
850
    @classmethod
851
852
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
853
            raise ValueError("Stream options can only be defined when `stream=True`.")
854
855
856
857
858
859
860

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
861
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
862
                raise ValueError(
863
864
                    "`prompt_logprobs` are not available when `stream=True`."
                )
865

866
            if prompt_logprobs < 0 and prompt_logprobs != -1:
867
                raise ValueError("`prompt_logprobs` must be a positive value or -1.")
868
        if (top_logprobs := data.get("top_logprobs")) is not None:
869
            if top_logprobs < 0 and top_logprobs != -1:
870
                raise ValueError("`top_logprobs` must be a positive value or -1.")
871

872
            if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"):
873
874
875
876
877
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )

        return data
878

879
880
    @model_validator(mode="before")
    @classmethod
881
    def check_structured_outputs_count(cls, data):
882
883
884
        if isinstance(data, ValueError):
            raise data

885
        if data.get("structured_outputs", None) is None:
886
887
            return data

888
        structured_outputs_kwargs = data["structured_outputs"]
889
890
        count = sum(
            structured_outputs_kwargs.get(k) is not None
891
892
            for k in ("json", "regex", "choice")
        )
893
894
        # you can only use one kind of constraints for structured outputs
        if count > 1:
895
            raise ValueError(
896
                "You can only use one kind of constraints for structured "
897
898
                "outputs ('json', 'regex' or 'choice')."
            )
899
900
        # you can only either use structured outputs or tools, not both
        if count > 1 and data.get("tool_choice", "none") not in (
901
902
903
            "none",
            "auto",
            "required",
904
        ):
905
            raise ValueError(
906
                "You can only either use constraints for structured outputs "
907
908
                "or tools, not both."
            )
909
910
911
912
        return data

    @model_validator(mode="before")
    @classmethod
913
914
915
    def check_tool_usage(cls, data):
        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
916
        if "tool_choice" not in data and data.get("tools"):
917
918
            data["tool_choice"] = "auto"

919
        # if "tool_choice" is "none" -- no validation is needed for tools
920
921
922
        if "tool_choice" in data and data["tool_choice"] == "none":
            return data

923
        # if "tool_choice" is specified -- validation
924
        if "tool_choice" in data and data["tool_choice"] is not None:
925
            # ensure that if "tool choice" is specified, tools are present
926
            if "tools" not in data or data["tools"] is None:
927
                raise ValueError("When using `tool_choice`, `tools` must be set.")
928
929

            # make sure that tool choice is either a named tool
930
            # OR that it's set to "auto" or "required"
931
932
933
            if data["tool_choice"] not in ["auto", "required"] and not isinstance(
                data["tool_choice"], dict
            ):
934
                raise ValueError(
935
936
937
                    f"Invalid value for `tool_choice`: {data['tool_choice']}! "
                    'Only named tools, "none", "auto" or "required" '
                    "are supported."
938
                )
939

940
941
942
            # if tool_choice is "required" but the "tools" list is empty,
            # override the data to behave like "none" to align with
            # OpenAI’s behavior.
943
944
945
946
947
            if (
                data["tool_choice"] == "required"
                and isinstance(data["tools"], list)
                and len(data["tools"]) == 0
            ):
948
949
950
951
                data["tool_choice"] = "none"
                del data["tools"]
                return data

952
953
            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
954
955
            correct_usage_message = (
                'Correct usage: `{"type": "function",'
956
                ' "function": {"name": "my_function"}}`'
957
            )
958
959
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
960
961
                function = data["tool_choice"].get("function")
                if not isinstance(function, dict):
962
                    raise ValueError(
963
                        f"Invalid value for `function`: `{function}` in "
964
965
                        f"`tool_choice`! {correct_usage_message}"
                    )
966
                if "name" not in function:
967
968
969
970
                    raise ValueError(
                        f"Expected field `name` in `function` in "
                        f"`tool_choice`! {correct_usage_message}"
                    )
971
                function_name = function["name"]
972
                if not isinstance(function_name, str) or len(function_name) == 0:
973
                    raise ValueError(
974
                        f"Invalid `name` in `function`: `{function_name}`"
975
976
                        f" in `tool_choice`! {correct_usage_message}"
                    )
977
                for tool in data["tools"]:
978
                    if tool["function"]["name"] == function_name:
979
980
981
982
983
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
984
985
                        " of the specified `tools`"
                    )
986
987
        return data

988
989
990
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
991
992
993
994
995
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
996
997
        return data

998
999
1000
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
1001
1002
1003
1004
1005
1006
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
1007
1008
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1009

1010
class CompletionRequest(OpenAIBaseModel):
1011
1012
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
1013
1014
1015
1016
1017
1018
1019
    model: str | None = None
    prompt: list[int] | list[list[int]] | str | list[str] | None = None
    echo: bool | None = False
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: int | None = None
    max_tokens: int | None = 16
1020
    n: int = 1
1021
1022
1023
1024
1025
1026
1027
1028
1029
    presence_penalty: float | None = 0.0
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    suffix: str | None = None
    temperature: float | None = None
    top_p: float | None = None
    user: str | None = None
1030

1031
    # --8<-- [start:completion-sampling-params]
1032
    use_beam_search: bool = False
1033
1034
1035
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
1036
    length_penalty: float = 1.0
1037
    stop_token_ids: list[int] | None = []
1038
1039
1040
1041
1042
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
1043
1044
1045
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    allowed_token_ids: list[int] | None = None
    prompt_logprobs: int | None = None
1046
    # --8<-- [end:completion-sampling-params]
1047

1048
    # --8<-- [start:completion-extra-params]
1049
    prompt_embeds: bytes | list[bytes] | None = None
1050
1051
    add_special_tokens: bool = Field(
        default=True,
1052
        description=(
1053
            "If true (the default), special tokens (e.g. BOS) will be added to "
1054
1055
            "the prompt."
        ),
1056
    )
1057
    response_format: AnyResponseFormat | None = Field(
1058
        default=None,
1059
1060
1061
1062
1063
        description=(
            "Similar to chat completion, this parameter specifies the format "
            "of output. Only {'type': 'json_object'}, {'type': 'json_schema'}"
            ", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
        ),
1064
    )
1065
    structured_outputs: StructuredOutputsParams | None = Field(
1066
        default=None,
1067
        description="Additional kwargs for structured outputs",
1068
    )
1069
1070
1071
1072
1073
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1074
1075
            "if the served model does not use priority scheduling."
        ),
1076
    )
1077
    request_id: str = Field(
1078
        default_factory=random_uuid,
1079
1080
1081
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1082
1083
            "through out the inference process and return in response."
        ),
1084
    )
1085
    logits_processors: LogitsProcessors | None = Field(
1086
1087
1088
1089
1090
1091
1092
1093
1094
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
1095
1096
1097
            "{'param': 'value'}}."
        ),
    )
1098

1099
    return_tokens_as_token_ids: bool | None = Field(
1100
1101
1102
1103
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
1104
1105
1106
            "that are not JSON-encodable can be identified."
        ),
    )
1107
    return_token_ids: bool | None = Field(
1108
1109
1110
1111
1112
1113
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
1114
1115
1116
            "need to map generated text back to input tokens."
        ),
    )
1117

1118
    cache_salt: str | None = Field(
1119
1120
1121
1122
1123
1124
1125
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
1126
            "to 256 bit)."
1127
1128
        ),
    )
1129

1130
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
1131
        default=None,
1132
1133
        description="KVTransfer parameters used for disaggregated serving.",
    )
Robert Shaw's avatar
Robert Shaw committed
1134

1135
    vllm_xargs: dict[str, str | int | float] | None = Field(
1136
        default=None,
1137
1138
1139
1140
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
1141
1142
    )

1143
    # --8<-- [end:completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
1144

1145
1146
1147
1148
1149
    # Default sampling parameters for completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
1150
        "top_k": 0,
1151
1152
1153
1154
        "min_p": 0.0,
    }

    def to_beam_search_params(
1155
1156
        self,
        max_tokens: int,
1157
        default_sampling_params: dict | None = None,
1158
1159
1160
    ) -> BeamSearchParams:
        if default_sampling_params is None:
            default_sampling_params = {}
1161
        n = self.n if self.n is not None else 1
1162
1163
1164

        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get("temperature", 1.0)
1165
1166
1167
1168
1169
1170

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
1171
            length_penalty=self.length_penalty,
1172
1173
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
1174

1175
    def to_sampling_params(
1176
        self,
1177
        max_tokens: int,
1178
1179
        logits_processor_pattern: str | None,
        default_sampling_params: dict | None = None,
1180
    ) -> SamplingParams:
1181
1182
        if default_sampling_params is None:
            default_sampling_params = {}
1183

1184
1185
1186
1187
1188
1189
1190
1191
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
1192
1193
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
1194
1195
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
1196
1197
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
1198
1199
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
1200
1201
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
1202
1203
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
1204
1205
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
1206

1207
1208
1209
1210
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

1211
1212
        echo_without_generation = self.echo and self.max_tokens == 0

1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
        response_format = self.response_format
        if response_format is not None:
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
            if response_format.type == "json_object":
                self.structured_outputs.json_object = True
            elif response_format.type == "json_schema":
                json_schema = response_format.json_schema
1225
                assert json_schema is not None
1226
1227
1228
                self.structured_outputs.json = json_schema.json_schema
            elif response_format.type == "structural_tag":
                structural_tag = response_format
1229
                assert structural_tag is not None and isinstance(
1230
1231
1232
1233
1234
                    structural_tag,
                    (
                        LegacyStructuralTagResponseFormat,
                        StructuralTagResponseFormat,
                    ),
1235
1236
                )
                s_tag_obj = structural_tag.model_dump(by_alias=True)
1237
                self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
1238

1239
1240
1241
1242
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
1243
        return SamplingParams.from_optional(
1244
1245
1246
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
1247
1248
1249
1250
1251
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
1252
            seed=self.seed,
1253
1254
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
1255
            logprobs=self.logprobs,
1256
            ignore_eos=self.ignore_eos,
1257
            max_tokens=max_tokens if not echo_without_generation else 1,
1258
            min_tokens=self.min_tokens,
1259
            prompt_logprobs=prompt_logprobs,
1260
            skip_special_tokens=self.skip_special_tokens,
1261
            spaces_between_special_tokens=self.spaces_between_special_tokens,
1262
            include_stop_str_in_output=self.include_stop_str_in_output,
1263
1264
1265
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
1266
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1267
1268
1269
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
1270
            structured_outputs=self.structured_outputs,
1271
            logit_bias=self.logit_bias,
Robert Shaw's avatar
Robert Shaw committed
1272
            allowed_token_ids=self.allowed_token_ids,
1273
            extra_args=extra_args or None,
1274
        )
1275

1276
1277
    @model_validator(mode="before")
    @classmethod
1278
    def check_structured_outputs_count(cls, data):
1279
        if data.get("structured_outputs", None) is None:
1280
1281
            return data

1282
        structured_outputs_kwargs = data["structured_outputs"]
1283
1284
        count = sum(
            structured_outputs_kwargs.get(k) is not None
1285
1286
            for k in ("json", "regex", "choice")
        )
1287
        if count > 1:
1288
            raise ValueError(
1289
                "You can only use one kind of constraints for structured "
1290
1291
                "outputs ('json', 'regex' or 'choice')."
            )
1292
1293
        return data

1294
1295
1296
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
1297
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
1298
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
1299
                raise ValueError(
1300
1301
                    "`prompt_logprobs` are not available when `stream=True`."
                )
1302

1303
            if prompt_logprobs < 0 and prompt_logprobs != -1:
1304
                raise ValueError("`prompt_logprobs` must be a positive value or -1.")
1305
1306
1307
        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
            raise ValueError("`logprobs` must be a positive value.")

1308
1309
        return data

1310
1311
1312
1313
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
1314
            raise ValueError("Stream options can only be defined when `stream=True`.")
1315

1316
1317
        return data

1318
1319
1320
    @model_validator(mode="before")
    @classmethod
    def validate_prompt_and_prompt_embeds(cls, data):
1321
1322
1323
        prompt = data.get("prompt")
        prompt_embeds = data.get("prompt_embeds")

1324
1325
1326
1327
        prompt_is_empty = prompt is None or (isinstance(prompt, str) and prompt == "")
        embeds_is_empty = prompt_embeds is None or (
            isinstance(prompt_embeds, list) and len(prompt_embeds) == 0
        )
1328
1329

        if prompt_is_empty and embeds_is_empty:
1330
            raise ValueError(
1331
1332
1333
                "Either prompt or prompt_embeds must be provided and non-empty."
            )

1334
1335
        return data

1336
1337
1338
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
1339
1340
1341
1342
1343
1344
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
1345
1346
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1347

1348
class EmbeddingCompletionRequest(OpenAIBaseModel):
1349
1350
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
1351
1352
    model: str | None = None
    input: list[int] | list[list[int]] | str | list[str]
1353
    encoding_format: EncodingFormat = "float"
1354
1355
1356
    dimensions: int | None = None
    user: str | None = None
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1357

1358
    # --8<-- [start:embedding-extra-params]
1359
1360
1361
1362
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
1363
1364
            "the prompt."
        ),
1365
    )
1366
1367
1368
1369
1370
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1371
1372
            "if the served model does not use priority scheduling."
        ),
1373
    )
1374
    request_id: str = Field(
1375
        default_factory=random_uuid,
1376
1377
1378
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1379
1380
            "through out the inference process and return in response."
        ),
1381
    )
1382
1383
1384
1385
    normalize: bool | None = Field(
        default=None,
        description="Whether to normalize the embeddings outputs. Default is True.",
    )
1386
    embed_dtype: EmbedDType = Field(
1387
1388
        default="float32",
        description=(
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
            "What dtype to use for encoding. Default to using float32 for base64 "
            "encoding to match the OpenAI python client behavior. "
            "This parameter will affect base64 and binary_response."
        ),
    )
    endianness: Endianness = Field(
        default="native",
        description=(
            "What endianness to use for encoding. Default to using native for "
            "base64 encoding to match the OpenAI python client behavior."
            "This parameter will affect base64 and binary_response."
1400
1401
        ),
    )
1402
    # --8<-- [end:embedding-extra-params]
1403

1404
    def to_pooling_params(self):
1405
1406
1407
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
1408
1409
            normalize=self.normalize,
        )
1410
1411


1412
class EmbeddingChatRequest(OpenAIBaseModel):
1413
    model: str | None = None
1414
    messages: list[ChatCompletionMessageParam]
1415

1416
    encoding_format: EncodingFormat = "float"
1417
1418
1419
    dimensions: int | None = None
    user: str | None = None
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1420

1421
    # --8<-- [start:chat-embedding-extra-params]
1422
1423
    add_generation_prompt: bool = Field(
        default=False,
1424
1425
1426
1427
1428
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
1429
1430
    )

1431
1432
1433
1434
1435
1436
1437
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
1438
1439
            "default)."
        ),
1440
    )
1441
    chat_template: str | None = Field(
1442
1443
1444
1445
1446
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
1447
1448
            "does not define one."
        ),
1449
    )
1450
    chat_template_kwargs: dict[str, Any] | None = Field(
1451
        default=None,
1452
1453
        description=(
            "Additional keyword args to pass to the template renderer. "
1454
1455
            "Will be accessible by the chat template."
        ),
1456
    )
1457
    mm_processor_kwargs: dict[str, Any] | None = Field(
1458
1459
1460
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
1461
1462
1463
1464
1465
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1466
1467
            "if the served model does not use priority scheduling."
        ),
1468
    )
1469
    request_id: str = Field(
1470
        default_factory=random_uuid,
1471
1472
1473
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1474
1475
            "through out the inference process and return in response."
        ),
1476
    )
1477
1478
1479
1480
    normalize: bool | None = Field(
        default=None,
        description="Whether to normalize the embeddings outputs. Default is True.",
    )
1481
    embed_dtype: EmbedDType = Field(
1482
1483
        default="float32",
        description=(
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
            "What dtype to use for encoding. Default to using float32 for base64 "
            "encoding to match the OpenAI python client behavior. "
            "This parameter will affect base64 and binary_response."
        ),
    )
    endianness: Endianness = Field(
        default="native",
        description=(
            "What endianness to use for encoding. Default to using native for "
            "base64 encoding to match the OpenAI python client behavior."
            "This parameter will affect base64 and binary_response."
1495
1496
        ),
    )
1497
    # --8<-- [end:chat-embedding-extra-params]
1498
1499
1500
1501

    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
1502
1503
1504
1505
1506
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
1507
1508
1509
        return data

    def to_pooling_params(self):
1510
1511
1512
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
1513
1514
            normalize=self.normalize,
        )
1515
1516


1517
EmbeddingRequest: TypeAlias = EmbeddingCompletionRequest | EmbeddingChatRequest
1518

1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570

class PoolingCompletionRequest(EmbeddingCompletionRequest):
    task: PoolingTask | None = None
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )
    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )
    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "If it is a classify or token_classify task, the default is True; "
        "for other tasks, this value should be None.",
    )

    def to_pooling_params(self):
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
            normalize=self.normalize,
            use_activation=get_use_activation(self),
        )


class PoolingChatRequest(EmbeddingChatRequest):
    task: PoolingTask | None = None
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )
    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )
    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "If it is a classify or token_classify task, the default is True; "
        "for other tasks, this value should be None.",
    )

    def to_pooling_params(self):
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
            normalize=self.normalize,
            use_activation=get_use_activation(self),
        )

1571
1572
1573
1574
1575

T = TypeVar("T")


class IOProcessorRequest(OpenAIBaseModel, Generic[T]):
1576
    model: str | None = None
1577
1578
1579
1580
1581
1582
1583
1584
1585

    priority: int = Field(default=0)
    """
    The priority of the request (lower means earlier handling;
    default: 0). Any priority other than 0 will raise an error
    if the served model does not use priority scheduling.
    """
    data: T

1586
    task: PoolingTask = "plugin"
1587
1588
    encoding_format: EncodingFormat = "float"
    embed_dtype: EmbedDType = Field(
1589
1590
        default="float32",
        description=(
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
            "What dtype to use for encoding. Default to using float32 for base64 "
            "encoding to match the OpenAI python client behavior. "
            "This parameter will affect base64 and binary_response."
        ),
    )
    endianness: Endianness = Field(
        default="native",
        description=(
            "What endianness to use for encoding. Default to using native for "
            "base64 encoding to match the OpenAI python client behavior."
            "This parameter will affect base64 and binary_response."
1602
1603
1604
        ),
    )

1605
    def to_pooling_params(self):
1606
        return PoolingParams()
1607
1608
1609


class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
1610
    request_id: str | None = None
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
    """
    The request_id associated with this response
    """
    created_at: int = Field(default_factory=lambda: int(time.time()))

    data: T
    """
    When using plugins IOProcessor plugins, the actual output is generated
    by the plugin itself. Hence, we use a generic type for the response data
    """


1623
1624
1625
PoolingRequest: TypeAlias = (
    PoolingCompletionRequest | PoolingChatRequest | IOProcessorRequest
)
1626

1627

1628
class ScoreRequest(OpenAIBaseModel):
1629
1630
1631
1632
    model: str | None = None
    text_1: list[str] | str | ScoreMultiModalParam
    text_2: list[str] | str | ScoreMultiModalParam
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1633

1634
    # --8<-- [start:score-extra-params]
1635

1636
    mm_processor_kwargs: dict[str, Any] | None = Field(
1637
1638
1639
1640
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )

1641
1642
1643
1644
1645
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1646
1647
            "if the served model does not use priority scheduling."
        ),
1648
    )
1649

1650
1651
1652
1653
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )
1654

1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )

    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "Default is True.",
    )
1665
    # --8<-- [end:score-extra-params]
1666

1667
    def to_pooling_params(self):
1668
1669
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1670
            use_activation=get_use_activation(self),
1671
        )
1672
1673


1674
class RerankRequest(OpenAIBaseModel):
1675
1676
1677
    model: str | None = None
    query: str | ScoreMultiModalParam
    documents: list[str] | ScoreMultiModalParam
1678
    top_n: int = Field(default_factory=lambda: 0)
1679
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1680

1681
    # --8<-- [start:rerank-extra-params]
1682

1683
    mm_processor_kwargs: dict[str, Any] | None = Field(
1684
1685
1686
1687
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )

1688
1689
1690
1691
1692
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1693
1694
            "if the served model does not use priority scheduling."
        ),
1695
    )
1696

1697
1698
1699
1700
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )
1701

1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )

    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "Default is True.",
    )
1712
    # --8<-- [end:rerank-extra-params]
1713

1714
    def to_pooling_params(self):
1715
1716
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1717
            use_activation=get_use_activation(self),
1718
        )
1719
1720
1721


class RerankDocument(BaseModel):
1722
1723
    text: str | None = None
    multi_modal: ScoreContentPartParam | None = None
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739


class RerankResult(BaseModel):
    index: int
    document: RerankDocument
    relevance_score: float


class RerankUsage(BaseModel):
    total_tokens: int


class RerankResponse(OpenAIBaseModel):
    id: str
    model: str
    usage: RerankUsage
1740
    results: list[RerankResult]
1741
1742


1743
class CompletionLogProbs(OpenAIBaseModel):
1744
    text_offset: list[int] = Field(default_factory=list)
1745
    token_logprobs: list[float | None] = Field(default_factory=list)
1746
    tokens: list[str] = Field(default_factory=list)
1747
    top_logprobs: list[dict[str, float] | None] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
1748
1749


1750
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1751
1752
    index: int
    text: str
1753
1754
1755
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1756
1757
1758
1759
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1760
1761
            "including encountering the EOS token"
        ),
1762
    )
1763
1764
1765
    token_ids: list[int] | None = None  # For response
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None  # For prompt
Zhuohan Li's avatar
Zhuohan Li committed
1766
1767


1768
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1769
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
1770
    object: Literal["text_completion"] = "text_completion"
Zhuohan Li's avatar
Zhuohan Li committed
1771
1772
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1773
    choices: list[CompletionResponseChoice]
1774
1775
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1776
    usage: UsageInfo
1777
1778

    # vLLM-specific fields that are not in OpenAI spec
1779
    kv_transfer_params: dict[str, Any] | None = Field(
1780
1781
        default=None, description="KVTransfer parameters."
    )
Zhuohan Li's avatar
Zhuohan Li committed
1782
1783


1784
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1785
1786
    index: int
    text: str
1787
1788
1789
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1790
1791
1792
1793
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1794
1795
            "including encountering the EOS token"
        ),
1796
    )
1797
1798
    # not part of the OpenAI spec but for tracing the tokens
    # prompt tokens is put into choice to align with CompletionResponseChoice
1799
1800
    prompt_token_ids: list[int] | None = None
    token_ids: list[int] | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1801
1802


1803
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1804
1805
1806
1807
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1808
    choices: list[CompletionResponseStreamChoice]
1809
    usage: UsageInfo | None = Field(default=None)
1810
1811


1812
class EmbeddingResponseData(OpenAIBaseModel):
1813
1814
    index: int
    object: str = "embedding"
1815
    embedding: list[float] | str
1816
1817


1818
class EmbeddingResponse(OpenAIBaseModel):
1819
    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
1820
1821
1822
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1823
    data: list[EmbeddingResponseData]
1824
1825
1826
    usage: UsageInfo


1827
1828
1829
1830
1831
1832
class EmbeddingBytesResponse(OpenAIBaseModel):
    body: list[bytes]
    metadata: str
    media_type: str = "application/octet-stream"


1833
1834
1835
class PoolingResponseData(OpenAIBaseModel):
    index: int
    object: str = "pooling"
1836
    data: list[list[float]] | list[float] | str
1837
1838
1839
1840
1841
1842
1843


class PoolingResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"pool-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1844
    data: list[PoolingResponseData]
1845
1846
1847
    usage: UsageInfo


1848
1849
1850
1851
1852
1853
class PoolingBytesResponse(OpenAIBaseModel):
    body: list[bytes]
    metadata: str
    media_type: str = "application/octet-stream"


1854
1855
1856
class ScoreResponseData(OpenAIBaseModel):
    index: int
    object: str = "score"
1857
    score: float
1858
1859
1860
1861
1862
1863
1864


class ScoreResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1865
    data: list[ScoreResponseData]
1866
1867
1868
    usage: UsageInfo


1869
class ClassificationCompletionRequest(OpenAIBaseModel):
1870
1871
    model: str | None = None
    input: list[str] | str
1872
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1873
    user: str | None = None
1874

1875
    # --8<-- [start:classification-extra-params]
1876
1877
1878
1879
1880
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1881
1882
            "if the served model does not use priority scheduling."
        ),
1883
    )
1884
1885
1886
1887
1888
1889
1890
1891
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."
        ),
    )
    request_id: str = Field(
1892
        default_factory=random_uuid,
1893
1894
1895
1896
1897
1898
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
1899
1900
1901
1902
1903
1904
1905
1906
1907
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )

    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )
1908

1909
1910
1911
1912
1913
    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "Default is True.",
    )
1914
    # --8<-- [end:classification-extra-params]
1915
1916

    def to_pooling_params(self):
1917
1918
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1919
            use_activation=get_use_activation(self),
1920
        )
1921
1922


1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
class ClassificationChatRequest(OpenAIBaseModel):
    model: str | None = None
    messages: list[ChatCompletionMessageParam]
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    user: str | None = None

    # --8<-- [start:chat-classification-extra-params]
    add_generation_prompt: bool = Field(
        default=False,
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
    )

    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
            "default)."
        ),
    )

    chat_template: str | None = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."
        ),
    )

    chat_template_kwargs: dict[str, Any] | None = Field(
        default=None,
        description=(
            "Additional keyword args to pass to the template renderer. "
            "Will be accessible by the chat template."
        ),
    )

    mm_processor_kwargs: dict[str, Any] | None = Field(
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )

    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."
        ),
    )

    request_id: str = Field(
1983
        default_factory=random_uuid,
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )

    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )

    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "Default is True.",
    )
    # --8<-- [end:chat-classification-extra-params]

    def to_pooling_params(self):
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            use_activation=get_use_activation(self),
        )


ClassificationRequest: TypeAlias = (
    ClassificationCompletionRequest | ClassificationChatRequest
)


2019
2020
class ClassificationData(OpenAIBaseModel):
    index: int
2021
    label: str | None
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
    probs: list[float]
    num_classes: int


class ClassificationResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"classify-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    data: list[ClassificationData]
    usage: UsageInfo


2035
2036
2037
2038
2039
2040
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
2041
    id: str = Field(default_factory=make_tool_call_id)
2042
2043
2044
2045
    type: Literal["function"] = "function"
    function: FunctionCall


2046
class DeltaFunctionCall(BaseModel):
2047
2048
    name: str | None = None
    arguments: str | None = None
2049
2050
2051
2052


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
2053
2054
    id: str | None = None
    type: Literal["function"] | None = None
2055
    index: int
2056
    function: DeltaFunctionCall | None = None
2057
2058
2059
2060
2061
2062
2063


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
2064
    tool_calls: list[ToolCall]
2065
2066
2067

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
2068
    content: str | None = None
2069
2070


2071
class ChatMessage(OpenAIBaseModel):
2072
    role: str
2073
2074
2075
2076
2077
    content: str | None = None
    refusal: str | None = None
    annotations: OpenAIAnnotation | None = None
    audio: OpenAIChatCompletionAudio | None = None
    function_call: FunctionCall | None = None
2078
    tool_calls: list[ToolCall] = Field(default_factory=list)
2079

2080
    # vLLM-specific fields that are not in OpenAI spec
2081
    reasoning: str | None = None
2082
    reasoning_content: str | None = None
2083
2084
2085
2086
2087
2088
2089
    """Deprecated: use `reasoning` instead."""

    @model_validator(mode="after")
    def handle_deprecated_reasoning_content(self):
        """Copy reasoning to reasoning_content for backward compatibility."""
        self.reasoning_content = self.reasoning
        return self
2090

2091

2092
2093
2094
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
2095
    bytes: list[int] | None = None
2096
2097
2098


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
2099
2100
    # Workaround: redefine fields name cache so that it's not
    # shared with the super class.
2101
    field_names: ClassVar[set[str] | None] = None
2102
    top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
2103
2104
2105


class ChatCompletionLogProbs(OpenAIBaseModel):
2106
    content: list[ChatCompletionLogProbsContent] | None = None
2107
2108


2109
class ChatCompletionResponseChoice(OpenAIBaseModel):
2110
2111
    index: int
    message: ChatMessage
2112
    logprobs: ChatCompletionLogProbs | None = None
2113
    # per OpenAI spec this is the default
2114
    finish_reason: str | None = "stop"
2115
    # not part of the OpenAI spec but included in vLLM for legacy reasons
2116
    stop_reason: int | str | None = None
2117
2118
    # not part of the OpenAI spec but is useful for tracing the tokens
    # in agent scenarios
2119
    token_ids: list[int] | None = None
2120
2121


2122
class ChatCompletionResponse(OpenAIBaseModel):
2123
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
2124
    object: Literal["chat.completion"] = "chat.completion"
2125
2126
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
2127
    choices: list[ChatCompletionResponseChoice]
2128
2129
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
2130
    usage: UsageInfo
2131
2132

    # vLLM-specific fields that are not in OpenAI spec
2133
2134
2135
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None
    kv_transfer_params: dict[str, Any] | None = Field(
2136
2137
        default=None, description="KVTransfer parameters."
    )
2138
2139


2140
class DeltaMessage(OpenAIBaseModel):
2141
2142
    role: str | None = None
    content: str | None = None
2143
    reasoning: str | None = None
2144
    reasoning_content: str | None = None
2145
    """Deprecated: use `reasoning` instead."""
2146
    tool_calls: list[DeltaToolCall] = Field(default_factory=list)
2147

2148
2149
2150
2151
2152
2153
    @model_validator(mode="after")
    def handle_deprecated_reasoning_content(self):
        """Copy reasoning to reasoning_content for backward compatibility."""
        self.reasoning_content = self.reasoning
        return self

2154

2155
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
2156
2157
    index: int
    delta: DeltaMessage
2158
2159
2160
    logprobs: ChatCompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2161
    # not part of the OpenAI spec but for tracing the tokens
2162
    token_ids: list[int] | None = None
2163
2164


2165
class ChatCompletionStreamResponse(OpenAIBaseModel):
2166
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
2167
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
2168
2169
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
2170
    choices: list[ChatCompletionResponseStreamChoice]
2171
    usage: UsageInfo | None = Field(default=None)
2172
    # not part of the OpenAI spec but for tracing the tokens
2173
    prompt_token_ids: list[int] | None = None
2174
2175


2176
2177
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
2178
2179
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2180
2181
2182
2183
2184
2185
2186
2187


class TranscriptionStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
    object: Literal["transcription.chunk"] = "transcription.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranscriptionResponseStreamChoice]
2188
    usage: UsageInfo | None = Field(default=None)
2189
2190


2191
2192
class InputTokensDetails(OpenAIBaseModel):
    cached_tokens: int
2193
2194
    input_tokens_per_turn: list[int] = Field(default_factory=list)
    cached_tokens_per_turn: list[int] = Field(default_factory=list)
2195
2196
2197


class OutputTokensDetails(OpenAIBaseModel):
2198
2199
    reasoning_tokens: int = 0
    tool_output_tokens: int = 0
2200
2201
    output_tokens_per_turn: list[int] = Field(default_factory=list)
    tool_output_tokens_per_turn: list[int] = Field(default_factory=list)
2202
2203
2204
2205
2206
2207
2208
2209


class ResponseUsage(OpenAIBaseModel):
    input_tokens: int
    input_tokens_details: InputTokensDetails
    output_tokens: int
    output_tokens_details: OutputTokensDetails
    total_tokens: int
2210
2211


2212
2213
2214
2215
2216
2217
def serialize_message(msg):
    """
    Serializes a single message
    """
    if isinstance(msg, dict):
        return msg
2218
    elif hasattr(msg, "to_dict"):
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
        return msg.to_dict()
    else:
        # fallback to pyandic dump
        return msg.model_dump_json()


def serialize_messages(msgs):
    """
    Serializes multiple messages
    """
    return [serialize_message(msg) for msg in msgs] if msgs else None


2232
2233
2234
2235
class ResponsesResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
    created_at: int = Field(default_factory=lambda: int(time.time()))
    # error: Optional[ResponseError] = None
2236
2237
2238
    incomplete_details: IncompleteDetails | None = None
    instructions: str | None = None
    metadata: Metadata | None = None
2239
2240
    model: str
    object: Literal["response"] = "response"
2241
    output: list[ResponseOutputItem]
2242
2243
2244
2245
2246
2247
2248
    parallel_tool_calls: bool
    temperature: float
    tool_choice: ToolChoice
    tools: list[Tool]
    top_p: float
    background: bool
    max_output_tokens: int
2249
2250
2251
2252
    max_tool_calls: int | None = None
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
2253
2254
    service_tier: Literal["auto", "default", "flex", "scale", "priority"]
    status: ResponseStatus
2255
2256
    text: ResponseTextConfig | None = None
    top_logprobs: int | None = None
2257
    truncation: Literal["auto", "disabled"]
2258
2259
    usage: ResponseUsage | None = None
    user: str | None = None
2260

2261
2262
2263
2264
    # --8<-- [start:responses-extra-params]
    # These are populated when enable_response_messages is set to True
    # NOTE: custom serialization is needed
    # see serialize_input_messages and serialize_output_messages
2265
2266
    input_messages: list[ChatCompletionMessageParam] | None = None
    output_messages: list[ChatCompletionMessageParam] | None = None
2267
2268
2269
2270
2271
2272
2273
    # --8<-- [end:responses-extra-params]

    # NOTE: openAI harmony doesn't serialize TextContent properly,
    # TODO: this fixes for TextContent, but need to verify for tools etc
    # https://github.com/openai/harmony/issues/78
    @field_serializer("output_messages", when_used="json")
    def serialize_output_messages(self, msgs, _info):
2274
        return serialize_messages(msgs)
2275
2276
2277
2278
2279

    # NOTE: openAI harmony doesn't serialize TextContent properly, this fixes it
    # https://github.com/openai/harmony/issues/78
    @field_serializer("input_messages", when_used="json")
    def serialize_input_messages(self, msgs, _info):
2280
        return serialize_messages(msgs)
2281

2282
2283
2284
2285
2286
2287
2288
2289
2290
    @classmethod
    def from_request(
        cls,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
        model_name: str,
        created_time: int,
        output: list[ResponseOutputItem],
        status: ResponseStatus,
2291
2292
2293
        usage: ResponseUsage | None = None,
        input_messages: list[ChatCompletionMessageParam] | None = None,
        output_messages: list[ChatCompletionMessageParam] | None = None,
2294
    ) -> "ResponsesResponse":
2295
        incomplete_details: IncompleteDetails | None = None
2296
2297
        if status == "incomplete":
            incomplete_details = IncompleteDetails(reason="max_output_tokens")
2298
2299
2300
        # TODO: implement the other reason for incomplete_details,
        # which is content_filter
        # incomplete_details = IncompleteDetails(reason='content_filter')
2301
2302
2303
        return cls(
            id=request.request_id,
            created_at=created_time,
2304
            incomplete_details=incomplete_details,
2305
2306
2307
2308
            instructions=request.instructions,
            metadata=request.metadata,
            model=model_name,
            output=output,
2309
2310
            input_messages=input_messages,
            output_messages=output_messages,
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
            parallel_tool_calls=request.parallel_tool_calls,
            temperature=sampling_params.temperature,
            tool_choice=request.tool_choice,
            tools=request.tools,
            top_p=sampling_params.top_p,
            background=request.background,
            max_output_tokens=sampling_params.max_tokens,
            max_tool_calls=request.max_tool_calls,
            previous_response_id=request.previous_response_id,
            prompt=request.prompt,
            reasoning=request.reasoning,
            service_tier=request.service_tier,
            status=status,
            text=request.text,
            top_logprobs=sampling_params.logprobs,
            truncation=request.truncation,
            user=request.user,
            usage=usage,
        )


2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartDoneEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.done"]
    """The type of the event. Always `response.reasoning_part.done`."""


# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartAddedEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.added"]
    """The type of the event. Always `response.reasoning_part.added`."""


2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
# vLLM Streaming Events
# Note: we override the response type with the vLLM ResponsesResponse type
class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseInProgressEvent(OpenAIResponseInProgressEvent):
    response: ResponsesResponse  # type: ignore[override]


2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
StreamingResponsesResponse: TypeAlias = (
    ResponseCreatedEvent
    | ResponseInProgressEvent
    | ResponseCompletedEvent
    | ResponseOutputItemAddedEvent
    | ResponseOutputItemDoneEvent
    | ResponseContentPartAddedEvent
    | ResponseContentPartDoneEvent
    | ResponseReasoningTextDeltaEvent
    | ResponseReasoningTextDoneEvent
    | ResponseReasoningPartAddedEvent
    | ResponseReasoningPartDoneEvent
    | ResponseCodeInterpreterCallInProgressEvent
    | ResponseCodeInterpreterCallCodeDeltaEvent
    | ResponseWebSearchCallInProgressEvent
    | ResponseWebSearchCallSearchingEvent
    | ResponseWebSearchCallCompletedEvent
    | ResponseCodeInterpreterCallCodeDoneEvent
    | ResponseCodeInterpreterCallInterpretingEvent
    | ResponseCodeInterpreterCallCompletedEvent
)
2411

2412
2413
2414
BatchRequestInputBody: TypeAlias = (
    ChatCompletionRequest | EmbeddingRequest | ScoreRequest | RerankRequest
)
2415
2416


2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

2436
    # The parameters of the request.
2437
    body: BatchRequestInputBody
2438

2439
    @field_validator("body", mode="plain")
2440
2441
2442
    @classmethod
    def check_type_for_url(cls, value: Any, info: ValidationInfo):
        # Use url to disambiguate models
2443
        url: str = info.data["url"]
2444
2445
2446
2447
        if url == "/v1/chat/completions":
            return ChatCompletionRequest.model_validate(value)
        if url == "/v1/embeddings":
            return TypeAdapter(EmbeddingRequest).validate_python(value)
2448
        if url.endswith("/score"):
2449
            return ScoreRequest.model_validate(value)
2450
2451
2452
        if url.endswith("/rerank"):
            return RerankRequest.model_validate(value)
        return TypeAdapter(BatchRequestInputBody).validate_python(value)
2453

2454

2455
2456
2457
2458
2459
2460
2461
2462
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
2463
2464
2465
2466
2467
2468
2469
    body: (
        ChatCompletionResponse
        | EmbeddingResponse
        | ScoreResponse
        | RerankResponse
        | None
    ) = None
2470
2471


2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

2483
    response: BatchResponseData | None
2484
2485
2486

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
2487
    error: Any | None
2488
2489


2490
class TokenizeCompletionRequest(OpenAIBaseModel):
2491
    model: str | None = None
2492
2493
    prompt: str

2494
2495
2496
2497
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
2498
2499
            "the prompt."
        ),
2500
    )
2501
    return_token_strs: bool | None = Field(
2502
        default=False,
2503
2504
2505
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
2506
    )
2507
2508
2509


class TokenizeChatRequest(OpenAIBaseModel):
2510
    model: str | None = None
2511
    messages: list[ChatCompletionMessageParam]
2512

2513
2514
    add_generation_prompt: bool = Field(
        default=True,
2515
2516
2517
2518
2519
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
2520
    )
2521
    return_token_strs: bool | None = Field(
2522
        default=False,
2523
2524
2525
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
2526
    )
2527
2528
    continue_final_message: bool = Field(
        default=False,
2529
2530
2531
2532
2533
2534
2535
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
2536
2537
2538
2539
2540
2541
2542
2543
    )
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
2544
2545
            "default)."
        ),
2546
    )
2547
    chat_template: str | None = Field(
2548
2549
2550
2551
2552
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
2553
2554
            "does not define one."
        ),
2555
    )
2556
    chat_template_kwargs: dict[str, Any] | None = Field(
2557
        default=None,
2558
2559
        description=(
            "Additional keyword args to pass to the template renderer. "
2560
2561
            "Will be accessible by the chat template."
        ),
2562
    )
2563
    mm_processor_kwargs: dict[str, Any] | None = Field(
2564
2565
2566
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
2567
    tools: list[ChatCompletionToolsParam] | None = Field(
2568
2569
2570
        default=None,
        description=("A list of tools the model may call."),
    )
2571

2572
2573
2574
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
2575
2576
2577
2578
2579
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
2580
2581
        return data

2582

2583
TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest
2584
2585
2586
2587
2588


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
2589
    tokens: list[int]
2590
    token_strs: list[str] | None = None
2591
2592
2593


class DetokenizeRequest(OpenAIBaseModel):
2594
    model: str | None = None
2595
    tokens: list[int]
2596
2597
2598
2599


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
2600
2601


2602
2603
class TokenizerInfoResponse(OpenAIBaseModel):
    """
2604
    Response containing tokenizer configuration
2605
2606
2607
2608
2609
2610
2611
    equivalent to tokenizer_config.json
    """

    model_config = ConfigDict(extra="allow")
    tokenizer_class: str


2612
class LoadLoRAAdapterRequest(BaseModel):
2613
2614
2615
2616
    lora_name: str
    lora_path: str


2617
class UnloadLoRAAdapterRequest(BaseModel):
2618
    lora_name: str
2619
    lora_int_id: int | None = Field(default=None)
2620
2621
2622


## Protocols for Audio
2623
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
2624
2625
2626
2627


class TranscriptionRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
2628
    # https://platform.openai.com/docs/api-reference/audio/createTranscription
2629
2630
2631
2632
2633
2634
2635

    file: UploadFile
    """
    The audio file object (not file name) to transcribe, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

2636
    model: str | None = None
2637
2638
2639
    """ID of the model to use.
    """

2640
    language: str | None = None
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
    """The language of the input audio.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy and latency.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    ## TODO (varun) : Support if set to 0, certain thresholds are met !!

2664
    timestamp_granularities: list[Literal["word", "segment"]] = Field(
2665
2666
        alias="timestamp_granularities[]", default=[]
    )
2667
2668
2669
2670
2671
2672
2673
2674
    """The timestamp granularities to populate for this transcription.

    `response_format` must be set `verbose_json` to use timestamp granularities.
    Either or both of these options are supported: `word`, or `segment`. Note:
    There is no additional latency for segment timestamps, but generating word
    timestamps incurs additional latency.
    """

2675
    stream: bool | None = False
2676
    """When set, it will enable output to be streamed in a similar fashion
2677
    as the Chat Completion endpoint.
2678
    """
2679
    # --8<-- [start:transcription-extra-params]
2680
    # Flattened stream option to simplify form data.
2681
2682
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2683

2684
    vllm_xargs: dict[str, str | int | float] | None = Field(
2685
        default=None,
2686
2687
2688
2689
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
2690
    )
2691
    # --8<-- [end:transcription-extra-params]
2692

2693
    to_language: str | None = None
2694
2695
    """The language of the output audio we transcribe to.

2696
    Please note that this is not currently used by supported models at this
2697
2698
2699
    time, but it is a placeholder for future use, matching translation api.
    """

2700
    # --8<-- [start:transcription-sampling-params]
2701
2702
2703
2704
2705
2706
2707
2708
2709
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """

2710
    top_p: float | None = None
2711
    """Enables nucleus (top-p) sampling, where tokens are selected from the
2712
2713
2714
    smallest possible set whose cumulative probability exceeds `p`.
    """

2715
    top_k: int | None = None
2716
2717
    """Limits sampling to the `k` most probable tokens at each step."""

2718
    min_p: float | None = None
2719
    """Filters out tokens with a probability lower than `min_p`, ensuring a
2720
2721
2722
    minimum likelihood threshold during sampling.
    """

2723
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2724
2725
    """The seed to use for sampling."""

2726
    frequency_penalty: float | None = 0.0
2727
2728
    """The frequency penalty to use for sampling."""

2729
    repetition_penalty: float | None = None
2730
2731
    """The repetition penalty to use for sampling."""

2732
    presence_penalty: float | None = 0.0
2733
    """The presence penalty to use for sampling."""
2734
    # --8<-- [end:transcription-sampling-params]
2735

2736
2737
    # Default sampling parameters for transcription requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
2738
2739
2740
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
2741
        "top_k": 0,
2742
        "min_p": 0.0,
2743
2744
2745
    }

    def to_sampling_params(
2746
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2747
    ) -> SamplingParams:
2748
2749
2750
2751
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
2752

2753
2754
2755
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2756
2757
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2758
2759
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
2760
2761
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
2762
2763
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
2764
2765
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
2766
2767
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
2768
2769
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
2770
2771
2772
2773

        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )

        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=repetition_penalty,
            presence_penalty=self.presence_penalty,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
            extra_args=self.vllm_xargs,
        )
2792
2793
2794

    @model_validator(mode="before")
    @classmethod
2795
2796
2797
2798
2799
2800
2801
    def validate_transcription_request(cls, data):
        if isinstance(data.get("file"), str):
            raise HTTPException(
                status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
                detail="Expected 'file' to be a file-like object, not 'str'.",
            )

2802
2803
2804
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
2805
            raise ValueError("Stream options can only be defined when `stream=True`.")
2806
2807

        return data
2808
2809
2810


# Transcription response objects
2811
2812
2813
2814
2815
class TranscriptionUsageAudio(OpenAIBaseModel):
    type: Literal["duration"] = "duration"
    seconds: int


2816
2817
2818
class TranscriptionResponse(OpenAIBaseModel):
    text: str
    """The transcribed text."""
2819
    usage: TranscriptionUsageAudio
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870


class TranscriptionWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranscriptionSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

    avg_logprob: float
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

    compression_ratio: float
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

    no_speech_prob: float
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

2871
    tokens: list[int]
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
    """Array of token IDs for the text content."""


class TranscriptionResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The transcribed text."""

2885
    segments: list[TranscriptionSegment] | None = None
2886
2887
    """Segments of the transcribed text and their corresponding details."""

2888
    words: list[TranscriptionWord] | None = None
2889
    """Extracted words and their corresponding timestamps."""
2890
2891
2892
2893


class TranslationResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
2894
2895
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2896
2897
2898
2899
2900
2901
2902
2903


class TranslationStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
    object: Literal["translation.chunk"] = "translation.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranslationResponseStreamChoice]
2904
    usage: UsageInfo | None = Field(default=None)
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916


class TranslationRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/audio/createTranslation

    file: UploadFile
    """
    The audio file object (not file name) to translate, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

2917
    model: str | None = None
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
    """ID of the model to use.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    # TODO support additional sampling parameters
    # --8<-- [start:translation-sampling-params]
2937
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2938
2939
    """The seed to use for sampling."""

2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """
    # --8<-- [end:translation-sampling-params]

    # --8<-- [start:translation-extra-params]
2951
    language: str | None = None
2952
2953
2954
2955
2956
2957
2958
    """The language of the input audio we translate from.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy.
    """

2959
    to_language: str | None = None
2960
2961
2962
2963
2964
2965
2966
    """The language of the input audio we translate to.

    Please note that this is not supported by all models, refer to the specific
    model documentation for more details.
    For instance, Whisper only supports `to_language=en`.
    """

2967
    stream: bool | None = False
2968
    """Custom field not present in the original OpenAI definition. When set,
2969
    it will enable output to be streamed in a similar fashion as the Chat
2970
    Completion endpoint.
2971
2972
    """
    # Flattened stream option to simplify form data.
2973
2974
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2975
2976
2977
2978
2979
2980
2981
2982
    # --8<-- [end:translation-extra-params]

    # Default sampling parameters for translation requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "temperature": 0,
    }

    def to_sampling_params(
2983
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2984
    ) -> SamplingParams:
2985
2986
2987
2988
2989
2990
2991
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2992
2993
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2994

2995
2996
2997
2998
2999
3000
3001
3002
        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
        )
3003
3004
3005
3006
3007
3008
3009

    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
3010
            raise ValueError("Stream options can only be defined when `stream=True`.")
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083

        return data


# Translation response objects
class TranslationResponse(OpenAIBaseModel):
    text: str
    """The translated text."""


class TranslationWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranslationSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

    avg_logprob: float
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

    compression_ratio: float
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

    no_speech_prob: float
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

    tokens: list[int]
    """Array of token IDs for the text content."""


class TranslationResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The translated text."""

3084
    segments: list[TranslationSegment] | None = None
3085
3086
    """Segments of the translated text and their corresponding details."""

3087
    words: list[TranslationWord] | None = None
3088
    """Extracted words and their corresponding timestamps."""
3089
3090
3091
3092
3093


####### Tokens IN <> Tokens OUT #######
class GenerateRequest(BaseModel):
    request_id: str = Field(
3094
        default_factory=random_uuid,
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    token_ids: list[int]
    """The token ids to generate text from."""

    # features: MultiModalFeatureSpec
    # TODO (NickLucche): implement once Renderer work is completed
    features: str | None = None
    """The processed MM inputs for the model."""

    sampling_params: SamplingParams
    """The sampling parameters for the model."""

    model: str | None = None

    stream: bool | None = False
    stream_options: StreamOptions | None = None
    cache_salt: str | None = Field(
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
            "to 256 bit)."
        ),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."
        ),
    )
    kv_transfer_params: dict[str, Any] | None = Field(
        default=None,
        description="KVTransfer parameters used for disaggregated serving.",
    )


class GenerateResponseChoice(BaseModel):
    index: int
    logprobs: ChatCompletionLogProbs | None = None
    # per OpenAI spec this is the default
    finish_reason: str | None = "stop"
    token_ids: list[int] | None = None


class GenerateResponse(BaseModel):
    request_id: str = Field(
3151
        default_factory=random_uuid,
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    choices: list[GenerateResponseChoice]

    prompt_logprobs: list[dict[int, Logprob] | None] | None = None

    kv_transfer_params: dict[str, Any] | None = Field(
        default=None,
        description="KVTransfer parameters used for disaggregated serving.",
    )