protocol.py 108 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import json
Zhuohan Li's avatar
Zhuohan Li committed
7
import time
8
from http import HTTPStatus
9
from typing import Annotated, Any, ClassVar, Generic, Literal, TypeAlias, TypeVar
Zhuohan Li's avatar
Zhuohan Li committed
10

11
import regex as re
12
import torch
13
from fastapi import HTTPException, UploadFile
14
from openai.types.chat.chat_completion_audio import (
15
16
17
    ChatCompletionAudio as OpenAIChatCompletionAudio,
)
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
18
19
20
21
22
from openai.types.responses import (
    ResponseCodeInterpreterCallCodeDeltaEvent,
    ResponseCodeInterpreterCallCodeDoneEvent,
    ResponseCodeInterpreterCallCompletedEvent,
    ResponseCodeInterpreterCallInProgressEvent,
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
    ResponseCodeInterpreterCallInterpretingEvent,
    ResponseContentPartAddedEvent,
    ResponseContentPartDoneEvent,
    ResponseFunctionToolCall,
    ResponseInputItemParam,
    ResponseOutputItem,
    ResponseOutputItemAddedEvent,
    ResponseOutputItemDoneEvent,
    ResponsePrompt,
    ResponseReasoningItem,
    ResponseReasoningTextDeltaEvent,
    ResponseReasoningTextDoneEvent,
    ResponseStatus,
    ResponseWebSearchCallCompletedEvent,
    ResponseWebSearchCallInProgressEvent,
    ResponseWebSearchCallSearchingEvent,
)
40
from openai.types.responses import (
41
42
43
    ResponseCompletedEvent as OpenAIResponseCompletedEvent,
)
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
44
from openai.types.responses import (
45
46
    ResponseInProgressEvent as OpenAIResponseInProgressEvent,
)
47
from openai.types.responses.response_reasoning_item import (
48
49
    Content as ResponseReasoningTextContent,
)
50
from openai_harmony import Message as OpenAIHarmonyMessage
51

52
53
from vllm.config.pooler import get_use_activation
from vllm.tasks import PoolingTask
54
55
56
57
58
59
from vllm.utils.serial_utils import (
    EmbedDType,
    EncodingFormat,
    Endianness,
)

60
61
62
63
# Backward compatibility for OpenAI client versions
try:  # For older openai versions (< 1.100.0)
    from openai.types.responses import ResponseTextConfig
except ImportError:  # For newer openai versions (>= 1.100.0)
64
    from openai.types.responses import ResponseFormatTextConfig as ResponseTextConfig
65

66

67
from openai.types.responses.response import IncompleteDetails, ToolChoice
68
69
from openai.types.responses.tool import Tool
from openai.types.shared import Metadata, Reasoning
70
71
72
73
74
from pydantic import (
    BaseModel,
    ConfigDict,
    Field,
    TypeAdapter,
75
    ValidationError,
76
    ValidationInfo,
77
    field_serializer,
78
79
80
    field_validator,
    model_validator,
)
Zhuohan Li's avatar
Zhuohan Li committed
81

82
83
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam, make_tool_call_id
from vllm.entrypoints.score_utils import ScoreContentPartParam, ScoreMultiModalParam
84
from vllm.logger import init_logger
85
from vllm.logprobs import Logprob
86
from vllm.pooling_params import PoolingParams
87
88
89
90
91
92
from vllm.sampling_params import (
    BeamSearchParams,
    RequestOutputKind,
    SamplingParams,
    StructuredOutputsParams,
)
93
94
from vllm.utils import random_uuid
from vllm.utils.import_utils import resolve_obj_by_qualname
95

96
97
logger = init_logger(__name__)

98
_LONG_INFO = torch.iinfo(torch.long)
99

Zhuohan Li's avatar
Zhuohan Li committed
100

101
class OpenAIBaseModel(BaseModel):
102
103
104
    # OpenAI API does allow extra fields
    model_config = ConfigDict(extra="allow")

105
    # Cache class field names
106
    field_names: ClassVar[set[str] | None] = None
107

108
    @model_validator(mode="wrap")
109
    @classmethod
110
111
112
113
    def __log_extra_fields__(cls, data, handler):
        result = handler(data)
        if not isinstance(data, dict):
            return result
114
115
        field_names = cls.field_names
        if field_names is None:
116
117
118
119
            # Get all class field names and their potential aliases
            field_names = set()
            for field_name, field in cls.model_fields.items():
                field_names.add(field_name)
120
                if alias := getattr(field, "alias", None):
121
122
123
124
125
126
                    field_names.add(alias)
            cls.field_names = field_names

        # Compare against both field names and aliases
        if any(k not in field_names for k in data):
            logger.warning(
127
                "The following fields were present in the request but ignored: %s",
128
129
                data.keys() - field_names,
            )
130
        return result
131
132


133
class ErrorInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
134
135
    message: str
    type: str
136
    param: str | None = None
137
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
138
139


140
141
142
143
class ErrorResponse(OpenAIBaseModel):
    error: ErrorInfo


144
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
145
146
147
148
149
150
151
152
153
154
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
155
    group: str | None = None
156
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
157
158


159
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
160
161
162
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
163
    owned_by: str = "vllm"
164
165
166
    root: str | None = None
    parent: str | None = None
    max_model_len: int | None = None
167
    permission: list[ModelPermission] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
168
169


170
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
171
    object: str = "list"
172
    data: list[ModelCard] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
173
174


175
class PromptTokenUsageInfo(OpenAIBaseModel):
176
    cached_tokens: int | None = None
177
178


179
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
180
181
    prompt_tokens: int = 0
    total_tokens: int = 0
182
183
    completion_tokens: int | None = 0
    prompt_tokens_details: PromptTokenUsageInfo | None = None
Zhuohan Li's avatar
Zhuohan Li committed
184
185


186
187
class RequestResponseMetadata(BaseModel):
    request_id: str
188
    final_usage_info: UsageInfo | None = None
189
190


191
192
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
193
    description: str | None = None
194
195
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
196
197
    json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
    strict: bool | None = None
198
199


200
class LegacyStructuralTag(OpenAIBaseModel):
201
202
203
    begin: str
    # schema is the field, but that causes conflicts with pydantic so
    # instead use structural_tag_schema with an alias
204
    structural_tag_schema: dict[str, Any] | None = Field(default=None, alias="schema")
205
206
207
    end: str


208
class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
209
    type: Literal["structural_tag"]
210
    structures: list[LegacyStructuralTag]
211
212
213
    triggers: list[str]


214
215
216
217
218
219
220
221
222
223
class StructuralTagResponseFormat(OpenAIBaseModel):
    type: Literal["structural_tag"]
    format: Any


AnyStructuralTagResponseFormat: TypeAlias = (
    LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
)


224
class ResponseFormat(OpenAIBaseModel):
225
    # type must be "json_schema", "json_object", or "text"
226
    type: Literal["text", "json_object", "json_schema"]
227
    json_schema: JsonSchemaResponseFormat | None = None
228
229


230
231
232
AnyResponseFormat: TypeAlias = (
    ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
)
233
234


235
class StreamOptions(OpenAIBaseModel):
236
237
    include_usage: bool | None = True
    continuous_usage_stats: bool | None = False
238
239


240
241
class FunctionDefinition(OpenAIBaseModel):
    name: str
242
243
    description: str | None = None
    parameters: dict[str, Any] | None = None
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


260
261
# extra="forbid" is a workaround to have kwargs as a field,
# see https://github.com/pydantic/pydantic/issues/3125
262
263
class LogitsProcessorConstructor(BaseModel):
    qualname: str
264
265
    args: list[Any] | None = None
    kwargs: dict[str, Any] | None = None
266

267
268
    model_config = ConfigDict(extra="forbid")

269

270
LogitsProcessors = list[str | LogitsProcessorConstructor]
271
272


273
def get_logits_processors(
274
275
    processors: LogitsProcessors | None, pattern: str | None
) -> list[Any] | None:
276
277
278
    if processors and pattern:
        logits_processors = []
        for processor in processors:
279
            qualname = processor if isinstance(processor, str) else processor.qualname
280
281
282
283
            if not re.match(pattern, qualname):
                raise ValueError(
                    f"Logits processor '{qualname}' is not allowed by this "
                    "server. See --logits-processor-pattern engine argument "
284
285
                    "for more information."
                )
286
287
288
289
290
291
292
            try:
                logits_processor = resolve_obj_by_qualname(qualname)
            except Exception as e:
                raise ValueError(
                    f"Logits processor '{qualname}' could not be resolved: {e}"
                ) from e
            if isinstance(processor, LogitsProcessorConstructor):
293
294
295
                logits_processor = logits_processor(
                    *processor.args or [], **processor.kwargs or {}
                )
296
297
298
299
300
            logits_processors.append(logits_processor)
        return logits_processors
    elif processors:
        raise ValueError(
            "The `logits_processors` argument is not supported by this "
301
            "server. See --logits-processor-pattern engine argument "
302
303
            "for more information."
        )
304
305
306
    return None


307
308
309
ResponseInputOutputItem: TypeAlias = (
    ResponseInputItemParam | ResponseReasoningItem | ResponseFunctionToolCall
)
310
311


312
313
314
class ResponsesRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/responses/create
315
316
    background: bool | None = False
    include: (
317
318
319
320
321
322
323
324
325
326
        list[
            Literal[
                "code_interpreter_call.outputs",
                "computer_call_output.output.image_url",
                "file_search_call.results",
                "message.input_image.image_url",
                "message.output_text.logprobs",
                "reasoning.encrypted_content",
            ],
        ]
327
328
329
330
331
332
333
334
335
336
337
338
        | None
    ) = None
    input: str | list[ResponseInputOutputItem]
    instructions: str | None = None
    max_output_tokens: int | None = None
    max_tool_calls: int | None = None
    metadata: Metadata | None = None
    model: str | None = None
    parallel_tool_calls: bool | None = True
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
339
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto"
340
341
342
343
    store: bool | None = True
    stream: bool | None = False
    temperature: float | None = None
    text: ResponseTextConfig | None = None
344
345
    tool_choice: ToolChoice = "auto"
    tools: list[Tool] = Field(default_factory=list)
346
347
348
349
    top_logprobs: int | None = 0
    top_p: float | None = None
    truncation: Literal["auto", "disabled"] | None = "disabled"
    user: str | None = None
350
351
352
353
354
355
356

    # --8<-- [start:responses-extra-params]
    request_id: str = Field(
        default_factory=lambda: f"resp_{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
357
358
            "through out the inference process and return in response."
        ),
359
    )
360
    mm_processor_kwargs: dict[str, Any] | None = Field(
361
362
363
364
365
366
367
368
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
369
370
            "if the served model does not use priority scheduling."
        ),
371
    )
372
    cache_salt: str | None = Field(
373
374
375
376
377
378
379
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
380
            "to 256 bit)."
381
382
        ),
    )
383
384
385
386
387

    enable_response_messages: bool = Field(
        default=False,
        description=(
            "Dictates whether or not to return messages as part of the "
388
            "response object. Currently only supported for"
389
390
391
            "non-background and gpt-oss only. "
        ),
    )
392
393
394
395
396
    # similar to input_messages / output_messages in ResponsesResponse
    # we take in previous_input_messages (ie in harmony format)
    # this cannot be used in conjunction with previous_response_id
    # TODO: consider supporting non harmony messages as well
    previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None
397
398
399
400
401
402
403
404
405
406
    # --8<-- [end:responses-extra-params]

    _DEFAULT_SAMPLING_PARAMS = {
        "temperature": 1.0,
        "top_p": 1.0,
    }

    def to_sampling_params(
        self,
        default_max_tokens: int,
407
        default_sampling_params: dict | None = None,
408
409
410
411
412
413
414
415
416
    ) -> SamplingParams:
        if self.max_output_tokens is None:
            max_tokens = default_max_tokens
        else:
            max_tokens = min(self.max_output_tokens, default_max_tokens)

        default_sampling_params = default_sampling_params or {}
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
417
418
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
419
420
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
421
422
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
423
        stop_token_ids = default_sampling_params.get("stop_token_ids")
424
425

        # Structured output
426
        structured_outputs = None
427
428
        if self.text is not None and self.text.format is not None:
            response_format = self.text.format
429
430
431
432
            if (
                response_format.type == "json_schema"
                and response_format.schema_ is not None
            ):
433
                structured_outputs = StructuredOutputsParams(
434
435
                    json=response_format.schema_
                )
436
437
438
439
440
441
442
443
            elif response_format.type == "json_object":
                raise NotImplementedError("json_object is not supported")

        # TODO: add more parameters
        return SamplingParams.from_optional(
            temperature=temperature,
            top_p=top_p,
            max_tokens=max_tokens,
444
            logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
445
            stop_token_ids=stop_token_ids,
446
447
448
            output_kind=(
                RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
            ),
449
            structured_outputs=structured_outputs,
450
451
        )

452
453
454
455
    def is_include_output_logprobs(self) -> bool:
        """Check if the request includes output logprobs."""
        if self.include is None:
            return False
456
457
458
459
        return (
            isinstance(self.include, list)
            and "message.output_text.logprobs" in self.include
        )
460

461
462
463
464
465
    @model_validator(mode="before")
    def validate_background(cls, data):
        if not data.get("background"):
            return data
        if not data.get("store", True):
466
            raise ValueError("background can only be used when `store` is true")
467
468
469
470
471
472
473
474
        return data

    @model_validator(mode="before")
    def validate_prompt(cls, data):
        if data.get("prompt") is not None:
            raise ValueError("prompt template is not supported")
        return data

475
476
    @model_validator(mode="before")
    def check_cache_salt_support(cls, data):
477
478
479
480
481
482
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
483
484
        return data

485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
    @model_validator(mode="before")
    def function_call_parsing(cls, data):
        """Parse function_call dictionaries into ResponseFunctionToolCall objects.
        This ensures Pydantic can properly resolve union types in the input field.
        Function calls provided as dicts are converted to ResponseFunctionToolCall
        objects before validation, while invalid structures are left for Pydantic
        to reject with appropriate error messages.
        """

        input_data = data.get("input")

        # Early return for None, strings, or bytes
        # (strings are iterable but shouldn't be processed)
        if input_data is None or isinstance(input_data, (str, bytes)):
            return data

        # Convert iterators (like ValidatorIterator) to list
        if not isinstance(input_data, list):
            try:
                input_data = list(input_data)
            except TypeError:
                # Not iterable, leave as-is for Pydantic to handle
                return data

        processed_input = []
        for item in input_data:
            if isinstance(item, dict) and item.get("type") == "function_call":
                try:
                    processed_input.append(ResponseFunctionToolCall(**item))
                except ValidationError:
                    # Let Pydantic handle validation for malformed function calls
                    logger.debug(
                        "Failed to parse function_call to ResponseFunctionToolCall, "
                        "leaving for Pydantic validation"
                    )
                    processed_input.append(item)
            else:
                processed_input.append(item)

        data["input"] = processed_input
        return data

527

528
class ChatCompletionRequest(OpenAIBaseModel):
529
530
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
531
    messages: list[ChatCompletionMessageParam]
532
533
534
535
536
537
    model: str | None = None
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: bool | None = False
    top_logprobs: int | None = 0
    max_tokens: int | None = Field(
538
        default=None,
539
540
        deprecated="max_tokens is deprecated in favor of "
        "the max_completion_tokens field",
541
    )
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
    max_completion_tokens: int | None = None
    n: int | None = 1
    presence_penalty: float | None = 0.0
    response_format: AnyResponseFormat | None = None
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    temperature: float | None = None
    top_p: float | None = None
    tools: list[ChatCompletionToolsParam] | None = None
    tool_choice: (
        Literal["none"]
        | Literal["auto"]
        | Literal["required"]
        | ChatCompletionNamedToolChoiceParam
        | None
    ) = "none"
    reasoning_effort: Literal["low", "medium", "high"] | None = None
561
    include_reasoning: bool = True
562

563
    # NOTE this will be ignored by vLLM -- the model determines the behavior
564
565
    parallel_tool_calls: bool | None = False
    user: str | None = None
566

567
    # --8<-- [start:chat-completion-sampling-params]
568
    use_beam_search: bool = False
569
570
571
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
572
    length_penalty: float = 1.0
573
    stop_token_ids: list[int] | None = []
574
575
576
577
578
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
579
580
581
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    prompt_logprobs: int | None = None
    allowed_token_ids: list[int] | None = None
582
    bad_words: list[str] = Field(default_factory=list)
583
    # --8<-- [end:chat-completion-sampling-params]
584

585
    # --8<-- [start:chat-completion-extra-params]
586
    echo: bool = Field(
587
588
589
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
590
591
            "if they belong to the same role."
        ),
592
    )
593
    add_generation_prompt: bool = Field(
594
        default=True,
595
596
597
598
599
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
600
    )
601
602
    continue_final_message: bool = Field(
        default=False,
603
604
605
606
607
608
609
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
610
    )
611
    add_special_tokens: bool = Field(
612
613
614
615
616
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
617
            "special tokens so this should be set to false (as is the "
618
619
            "default)."
        ),
620
    )
621
    documents: list[dict[str, str]] | None = Field(
622
        default=None,
623
624
625
626
627
628
629
        description=(
            "A list of dicts representing documents that will be accessible to "
            "the model if it is performing RAG (retrieval-augmented generation)."
            " If the template does not support RAG, this argument will have no "
            "effect. We recommend that each document should be a dict containing "
            '"title" and "text" keys.'
        ),
630
    )
631
    chat_template: str | None = Field(
632
633
634
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
635
636
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
637
638
            "does not define one."
        ),
639
    )
640
    chat_template_kwargs: dict[str, Any] | None = Field(
641
        default=None,
642
643
        description=(
            "Additional keyword args to pass to the template renderer. "
644
645
            "Will be accessible by the chat template."
        ),
646
    )
647
    mm_processor_kwargs: dict[str, Any] | None = Field(
648
649
650
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
651
    structured_outputs: StructuredOutputsParams | None = Field(
652
        default=None,
653
        description="Additional kwargs for structured outputs",
654
    )
655
656
657
658
659
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
660
661
            "if the served model does not use priority scheduling."
        ),
662
    )
663
664
665
666
667
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
668
669
            "through out the inference process and return in response."
        ),
670
    )
671
    logits_processors: LogitsProcessors | None = Field(
672
673
674
675
676
677
678
679
680
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
681
682
683
            "{'param': 'value'}}."
        ),
    )
684
    return_tokens_as_token_ids: bool | None = Field(
685
686
687
688
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
689
690
691
            "that are not JSON-encodable can be identified."
        ),
    )
692
    return_token_ids: bool | None = Field(
693
694
695
696
697
698
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
699
700
701
            "need to map generated text back to input tokens."
        ),
    )
702
    cache_salt: str | None = Field(
703
704
705
706
707
708
709
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
710
            "to 256 bit)."
711
712
        ),
    )
713
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
714
        default=None,
715
716
        description="KVTransfer parameters used for disaggregated serving.",
    )
717

718
    vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
719
        default=None,
720
        description=(
721
            "Additional request parameters with (list of) string or "
722
723
            "numeric values, used by custom extensions."
        ),
724
725
    )

726
    # --8<-- [end:chat-completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
727

728
729
730
731
732
    # Default sampling parameters for chat completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
733
        "top_k": 0,
734
735
736
737
        "min_p": 0.0,
    }

    def to_beam_search_params(
738
739
        self, max_tokens: int, default_sampling_params: dict
    ) -> BeamSearchParams:
740
        n = self.n if self.n is not None else 1
741
742
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
743
744
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
745
746
747
748
749
750

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
751
            length_penalty=self.length_penalty,
752
753
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
754

755
    def to_sampling_params(
756
        self,
757
        max_tokens: int,
758
        logits_processor_pattern: str | None,
759
        default_sampling_params: dict,
760
    ) -> SamplingParams:
761
762
763
764
765
766
767
768
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
769
770
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
771
772
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
773
774
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
775
776
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
777
778
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
779
780
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
781
782
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
783

784
785
786
787
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

788
        response_format = self.response_format
789
        if response_format is not None:
790
791
792
793
794
795
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
            if response_format.type == "json_object":
                self.structured_outputs.json_object = True
            elif response_format.type == "json_schema":
                json_schema = response_format.json_schema
                assert json_schema is not None
                self.structured_outputs.json = json_schema.json_schema
            elif response_format.type == "structural_tag":
                structural_tag = response_format
                assert structural_tag is not None and isinstance(
                    structural_tag,
                    (
                        LegacyStructuralTagResponseFormat,
                        StructuralTagResponseFormat,
                    ),
                )
                s_tag_obj = structural_tag.model_dump(by_alias=True)
                self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
813

814
815
816
817
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
818
        return SamplingParams.from_optional(
819
820
821
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
822
823
824
825
826
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
827
            seed=self.seed,
828
829
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
830
            logprobs=self.top_logprobs if self.logprobs else None,
831
            prompt_logprobs=prompt_logprobs,
832
            ignore_eos=self.ignore_eos,
833
            max_tokens=max_tokens,
834
            min_tokens=self.min_tokens,
835
836
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
837
838
839
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
840
            include_stop_str_in_output=self.include_stop_str_in_output,
841
            truncate_prompt_tokens=self.truncate_prompt_tokens,
842
843
844
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
845
            structured_outputs=self.structured_outputs,
Robert Shaw's avatar
Robert Shaw committed
846
            logit_bias=self.logit_bias,
847
            bad_words=self.bad_words,
848
            allowed_token_ids=self.allowed_token_ids,
849
850
            extra_args=extra_args or None,
        )
851

852
    @model_validator(mode="before")
853
    @classmethod
854
855
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
856
            raise ValueError("Stream options can only be defined when `stream=True`.")
857
858
859
860
861
862
863

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
864
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
865
                raise ValueError(
866
867
                    "`prompt_logprobs` are not available when `stream=True`."
                )
868

869
            if prompt_logprobs < 0 and prompt_logprobs != -1:
870
                raise ValueError("`prompt_logprobs` must be a positive value or -1.")
871
        if (top_logprobs := data.get("top_logprobs")) is not None:
872
            if top_logprobs < 0 and top_logprobs != -1:
873
                raise ValueError("`top_logprobs` must be a positive value or -1.")
874

875
            if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"):
876
877
878
879
880
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )

        return data
881

882
883
    @model_validator(mode="before")
    @classmethod
884
    def check_structured_outputs_count(cls, data):
885
886
887
        if isinstance(data, ValueError):
            raise data

888
        if data.get("structured_outputs", None) is None:
889
890
            return data

891
        structured_outputs_kwargs = data["structured_outputs"]
892
893
        count = sum(
            structured_outputs_kwargs.get(k) is not None
894
895
            for k in ("json", "regex", "choice")
        )
896
897
        # you can only use one kind of constraints for structured outputs
        if count > 1:
898
            raise ValueError(
899
                "You can only use one kind of constraints for structured "
900
901
                "outputs ('json', 'regex' or 'choice')."
            )
902
903
        # you can only either use structured outputs or tools, not both
        if count > 1 and data.get("tool_choice", "none") not in (
904
905
906
            "none",
            "auto",
            "required",
907
        ):
908
            raise ValueError(
909
                "You can only either use constraints for structured outputs "
910
911
                "or tools, not both."
            )
912
913
914
915
        return data

    @model_validator(mode="before")
    @classmethod
916
917
918
    def check_tool_usage(cls, data):
        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
919
        if "tool_choice" not in data and data.get("tools"):
920
921
            data["tool_choice"] = "auto"

922
        # if "tool_choice" is "none" -- no validation is needed for tools
923
924
925
        if "tool_choice" in data and data["tool_choice"] == "none":
            return data

926
        # if "tool_choice" is specified -- validation
927
        if "tool_choice" in data and data["tool_choice"] is not None:
928
            # ensure that if "tool choice" is specified, tools are present
929
            if "tools" not in data or data["tools"] is None:
930
                raise ValueError("When using `tool_choice`, `tools` must be set.")
931
932

            # make sure that tool choice is either a named tool
933
            # OR that it's set to "auto" or "required"
934
935
936
            if data["tool_choice"] not in ["auto", "required"] and not isinstance(
                data["tool_choice"], dict
            ):
937
                raise ValueError(
938
939
940
                    f"Invalid value for `tool_choice`: {data['tool_choice']}! "
                    'Only named tools, "none", "auto" or "required" '
                    "are supported."
941
                )
942

943
944
945
            # if tool_choice is "required" but the "tools" list is empty,
            # override the data to behave like "none" to align with
            # OpenAI’s behavior.
946
947
948
949
950
            if (
                data["tool_choice"] == "required"
                and isinstance(data["tools"], list)
                and len(data["tools"]) == 0
            ):
951
952
953
954
                data["tool_choice"] = "none"
                del data["tools"]
                return data

955
956
            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
957
958
            correct_usage_message = (
                'Correct usage: `{"type": "function",'
959
                ' "function": {"name": "my_function"}}`'
960
            )
961
962
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
963
964
                function = data["tool_choice"].get("function")
                if not isinstance(function, dict):
965
                    raise ValueError(
966
                        f"Invalid value for `function`: `{function}` in "
967
968
                        f"`tool_choice`! {correct_usage_message}"
                    )
969
                if "name" not in function:
970
971
972
973
                    raise ValueError(
                        f"Expected field `name` in `function` in "
                        f"`tool_choice`! {correct_usage_message}"
                    )
974
                function_name = function["name"]
975
                if not isinstance(function_name, str) or len(function_name) == 0:
976
                    raise ValueError(
977
                        f"Invalid `name` in `function`: `{function_name}`"
978
979
                        f" in `tool_choice`! {correct_usage_message}"
                    )
980
                for tool in data["tools"]:
981
                    if tool["function"]["name"] == function_name:
982
983
984
985
986
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
987
988
                        " of the specified `tools`"
                    )
989
990
        return data

991
992
993
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
994
995
996
997
998
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
999
1000
        return data

1001
1002
1003
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
1004
1005
1006
1007
1008
1009
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
1010
1011
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1012

1013
class CompletionRequest(OpenAIBaseModel):
1014
1015
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
1016
1017
1018
1019
1020
1021
1022
    model: str | None = None
    prompt: list[int] | list[list[int]] | str | list[str] | None = None
    echo: bool | None = False
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: int | None = None
    max_tokens: int | None = 16
1023
    n: int = 1
1024
1025
1026
1027
1028
1029
1030
1031
1032
    presence_penalty: float | None = 0.0
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    suffix: str | None = None
    temperature: float | None = None
    top_p: float | None = None
    user: str | None = None
1033

1034
    # --8<-- [start:completion-sampling-params]
1035
    use_beam_search: bool = False
1036
1037
1038
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
1039
    length_penalty: float = 1.0
1040
    stop_token_ids: list[int] | None = []
1041
1042
1043
1044
1045
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
1046
1047
1048
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    allowed_token_ids: list[int] | None = None
    prompt_logprobs: int | None = None
1049
    # --8<-- [end:completion-sampling-params]
1050

1051
    # --8<-- [start:completion-extra-params]
1052
    prompt_embeds: bytes | list[bytes] | None = None
1053
1054
    add_special_tokens: bool = Field(
        default=True,
1055
        description=(
1056
            "If true (the default), special tokens (e.g. BOS) will be added to "
1057
1058
            "the prompt."
        ),
1059
    )
1060
    response_format: AnyResponseFormat | None = Field(
1061
        default=None,
1062
1063
1064
1065
1066
        description=(
            "Similar to chat completion, this parameter specifies the format "
            "of output. Only {'type': 'json_object'}, {'type': 'json_schema'}"
            ", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
        ),
1067
    )
1068
    structured_outputs: StructuredOutputsParams | None = Field(
1069
        default=None,
1070
        description="Additional kwargs for structured outputs",
1071
    )
1072
1073
1074
1075
1076
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1077
1078
            "if the served model does not use priority scheduling."
        ),
1079
    )
1080
1081
1082
1083
1084
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1085
1086
            "through out the inference process and return in response."
        ),
1087
    )
1088
    logits_processors: LogitsProcessors | None = Field(
1089
1090
1091
1092
1093
1094
1095
1096
1097
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
1098
1099
1100
            "{'param': 'value'}}."
        ),
    )
1101

1102
    return_tokens_as_token_ids: bool | None = Field(
1103
1104
1105
1106
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
1107
1108
1109
            "that are not JSON-encodable can be identified."
        ),
    )
1110
    return_token_ids: bool | None = Field(
1111
1112
1113
1114
1115
1116
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
1117
1118
1119
            "need to map generated text back to input tokens."
        ),
    )
1120

1121
    cache_salt: str | None = Field(
1122
1123
1124
1125
1126
1127
1128
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
1129
            "to 256 bit)."
1130
1131
        ),
    )
1132

1133
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
1134
        default=None,
1135
1136
        description="KVTransfer parameters used for disaggregated serving.",
    )
Robert Shaw's avatar
Robert Shaw committed
1137

1138
    vllm_xargs: dict[str, str | int | float] | None = Field(
1139
        default=None,
1140
1141
1142
1143
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
1144
1145
    )

1146
    # --8<-- [end:completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
1147

1148
1149
1150
1151
1152
    # Default sampling parameters for completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
1153
        "top_k": 0,
1154
1155
1156
1157
        "min_p": 0.0,
    }

    def to_beam_search_params(
1158
1159
        self,
        max_tokens: int,
1160
        default_sampling_params: dict | None = None,
1161
1162
1163
    ) -> BeamSearchParams:
        if default_sampling_params is None:
            default_sampling_params = {}
1164
        n = self.n if self.n is not None else 1
1165
1166
1167

        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get("temperature", 1.0)
1168
1169
1170
1171
1172
1173

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
1174
            length_penalty=self.length_penalty,
1175
1176
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
1177

1178
    def to_sampling_params(
1179
        self,
1180
        max_tokens: int,
1181
1182
        logits_processor_pattern: str | None,
        default_sampling_params: dict | None = None,
1183
    ) -> SamplingParams:
1184
1185
        if default_sampling_params is None:
            default_sampling_params = {}
1186

1187
1188
1189
1190
1191
1192
1193
1194
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
1195
1196
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
1197
1198
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
1199
1200
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
1201
1202
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
1203
1204
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
1205
1206
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
1207
1208
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
1209

1210
1211
1212
1213
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

1214
1215
        echo_without_generation = self.echo and self.max_tokens == 0

1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
        response_format = self.response_format
        if response_format is not None:
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
            if response_format.type == "json_object":
                self.structured_outputs.json_object = True
            elif response_format.type == "json_schema":
                json_schema = response_format.json_schema
1228
                assert json_schema is not None
1229
1230
1231
                self.structured_outputs.json = json_schema.json_schema
            elif response_format.type == "structural_tag":
                structural_tag = response_format
1232
                assert structural_tag is not None and isinstance(
1233
1234
1235
1236
1237
                    structural_tag,
                    (
                        LegacyStructuralTagResponseFormat,
                        StructuralTagResponseFormat,
                    ),
1238
1239
                )
                s_tag_obj = structural_tag.model_dump(by_alias=True)
1240
                self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
1241

1242
1243
1244
1245
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
1246
        return SamplingParams.from_optional(
1247
1248
1249
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
1250
1251
1252
1253
1254
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
1255
            seed=self.seed,
1256
1257
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
1258
            logprobs=self.logprobs,
1259
            ignore_eos=self.ignore_eos,
1260
            max_tokens=max_tokens if not echo_without_generation else 1,
1261
            min_tokens=self.min_tokens,
1262
            prompt_logprobs=prompt_logprobs,
1263
            skip_special_tokens=self.skip_special_tokens,
1264
            spaces_between_special_tokens=self.spaces_between_special_tokens,
1265
            include_stop_str_in_output=self.include_stop_str_in_output,
1266
1267
1268
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
1269
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1270
1271
1272
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
1273
            structured_outputs=self.structured_outputs,
1274
            logit_bias=self.logit_bias,
Robert Shaw's avatar
Robert Shaw committed
1275
            allowed_token_ids=self.allowed_token_ids,
1276
            extra_args=extra_args or None,
1277
        )
1278

1279
1280
    @model_validator(mode="before")
    @classmethod
1281
    def check_structured_outputs_count(cls, data):
1282
        if data.get("structured_outputs", None) is None:
1283
1284
            return data

1285
        structured_outputs_kwargs = data["structured_outputs"]
1286
1287
        count = sum(
            structured_outputs_kwargs.get(k) is not None
1288
1289
            for k in ("json", "regex", "choice")
        )
1290
        if count > 1:
1291
            raise ValueError(
1292
                "You can only use one kind of constraints for structured "
1293
1294
                "outputs ('json', 'regex' or 'choice')."
            )
1295
1296
        return data

1297
1298
1299
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
1300
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
1301
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
1302
                raise ValueError(
1303
1304
                    "`prompt_logprobs` are not available when `stream=True`."
                )
1305

1306
            if prompt_logprobs < 0 and prompt_logprobs != -1:
1307
                raise ValueError("`prompt_logprobs` must be a positive value or -1.")
1308
1309
1310
        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
            raise ValueError("`logprobs` must be a positive value.")

1311
1312
        return data

1313
1314
1315
1316
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
1317
            raise ValueError("Stream options can only be defined when `stream=True`.")
1318

1319
1320
        return data

1321
1322
1323
    @model_validator(mode="before")
    @classmethod
    def validate_prompt_and_prompt_embeds(cls, data):
1324
1325
1326
        prompt = data.get("prompt")
        prompt_embeds = data.get("prompt_embeds")

1327
1328
1329
1330
        prompt_is_empty = prompt is None or (isinstance(prompt, str) and prompt == "")
        embeds_is_empty = prompt_embeds is None or (
            isinstance(prompt_embeds, list) and len(prompt_embeds) == 0
        )
1331
1332

        if prompt_is_empty and embeds_is_empty:
1333
            raise ValueError(
1334
1335
1336
                "Either prompt or prompt_embeds must be provided and non-empty."
            )

1337
1338
        return data

1339
1340
1341
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
1342
1343
1344
1345
1346
1347
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
1348
1349
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1350

1351
class EmbeddingCompletionRequest(OpenAIBaseModel):
1352
1353
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
1354
1355
    model: str | None = None
    input: list[int] | list[list[int]] | str | list[str]
1356
    encoding_format: EncodingFormat = "float"
1357
1358
1359
    dimensions: int | None = None
    user: str | None = None
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1360

1361
    # --8<-- [start:embedding-extra-params]
1362
1363
1364
1365
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
1366
1367
            "the prompt."
        ),
1368
    )
1369
1370
1371
1372
1373
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1374
1375
            "if the served model does not use priority scheduling."
        ),
1376
    )
1377
1378
1379
1380
1381
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1382
1383
            "through out the inference process and return in response."
        ),
1384
    )
1385
1386
1387
1388
    normalize: bool | None = Field(
        default=None,
        description="Whether to normalize the embeddings outputs. Default is True.",
    )
1389
    embed_dtype: EmbedDType = Field(
1390
1391
        default="float32",
        description=(
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
            "What dtype to use for encoding. Default to using float32 for base64 "
            "encoding to match the OpenAI python client behavior. "
            "This parameter will affect base64 and binary_response."
        ),
    )
    endianness: Endianness = Field(
        default="native",
        description=(
            "What endianness to use for encoding. Default to using native for "
            "base64 encoding to match the OpenAI python client behavior."
            "This parameter will affect base64 and binary_response."
1403
1404
        ),
    )
1405
    # --8<-- [end:embedding-extra-params]
1406

1407
    def to_pooling_params(self):
1408
1409
1410
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
1411
1412
            normalize=self.normalize,
        )
1413
1414


1415
class EmbeddingChatRequest(OpenAIBaseModel):
1416
    model: str | None = None
1417
    messages: list[ChatCompletionMessageParam]
1418

1419
    encoding_format: EncodingFormat = "float"
1420
1421
1422
    dimensions: int | None = None
    user: str | None = None
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1423

1424
    # --8<-- [start:chat-embedding-extra-params]
1425
1426
    add_generation_prompt: bool = Field(
        default=False,
1427
1428
1429
1430
1431
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
1432
1433
    )

1434
1435
1436
1437
1438
1439
1440
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
1441
1442
            "default)."
        ),
1443
    )
1444
    chat_template: str | None = Field(
1445
1446
1447
1448
1449
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
1450
1451
            "does not define one."
        ),
1452
    )
1453
    chat_template_kwargs: dict[str, Any] | None = Field(
1454
        default=None,
1455
1456
        description=(
            "Additional keyword args to pass to the template renderer. "
1457
1458
            "Will be accessible by the chat template."
        ),
1459
    )
1460
    mm_processor_kwargs: dict[str, Any] | None = Field(
1461
1462
1463
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
1464
1465
1466
1467
1468
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1469
1470
            "if the served model does not use priority scheduling."
        ),
1471
    )
1472
1473
1474
1475
1476
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1477
1478
            "through out the inference process and return in response."
        ),
1479
    )
1480
1481
1482
1483
    normalize: bool | None = Field(
        default=None,
        description="Whether to normalize the embeddings outputs. Default is True.",
    )
1484
    embed_dtype: EmbedDType = Field(
1485
1486
        default="float32",
        description=(
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
            "What dtype to use for encoding. Default to using float32 for base64 "
            "encoding to match the OpenAI python client behavior. "
            "This parameter will affect base64 and binary_response."
        ),
    )
    endianness: Endianness = Field(
        default="native",
        description=(
            "What endianness to use for encoding. Default to using native for "
            "base64 encoding to match the OpenAI python client behavior."
            "This parameter will affect base64 and binary_response."
1498
1499
        ),
    )
1500
    # --8<-- [end:chat-embedding-extra-params]
1501
1502
1503
1504

    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
1505
1506
1507
1508
1509
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
1510
1511
1512
        return data

    def to_pooling_params(self):
1513
1514
1515
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
1516
1517
            normalize=self.normalize,
        )
1518
1519


1520
EmbeddingRequest: TypeAlias = EmbeddingCompletionRequest | EmbeddingChatRequest
1521

1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573

class PoolingCompletionRequest(EmbeddingCompletionRequest):
    task: PoolingTask | None = None
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )
    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )
    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "If it is a classify or token_classify task, the default is True; "
        "for other tasks, this value should be None.",
    )

    def to_pooling_params(self):
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
            normalize=self.normalize,
            use_activation=get_use_activation(self),
        )


class PoolingChatRequest(EmbeddingChatRequest):
    task: PoolingTask | None = None
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )
    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )
    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "If it is a classify or token_classify task, the default is True; "
        "for other tasks, this value should be None.",
    )

    def to_pooling_params(self):
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            dimensions=self.dimensions,
            normalize=self.normalize,
            use_activation=get_use_activation(self),
        )

1574
1575
1576
1577
1578

T = TypeVar("T")


class IOProcessorRequest(OpenAIBaseModel, Generic[T]):
1579
    model: str | None = None
1580
1581
1582
1583
1584
1585
1586
1587
1588

    priority: int = Field(default=0)
    """
    The priority of the request (lower means earlier handling;
    default: 0). Any priority other than 0 will raise an error
    if the served model does not use priority scheduling.
    """
    data: T

1589
    task: PoolingTask = "plugin"
1590
1591
    encoding_format: EncodingFormat = "float"
    embed_dtype: EmbedDType = Field(
1592
1593
        default="float32",
        description=(
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
            "What dtype to use for encoding. Default to using float32 for base64 "
            "encoding to match the OpenAI python client behavior. "
            "This parameter will affect base64 and binary_response."
        ),
    )
    endianness: Endianness = Field(
        default="native",
        description=(
            "What endianness to use for encoding. Default to using native for "
            "base64 encoding to match the OpenAI python client behavior."
            "This parameter will affect base64 and binary_response."
1605
1606
1607
        ),
    )

1608
    def to_pooling_params(self):
1609
        return PoolingParams()
1610
1611
1612


class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
1613
    request_id: str | None = None
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
    """
    The request_id associated with this response
    """
    created_at: int = Field(default_factory=lambda: int(time.time()))

    data: T
    """
    When using plugins IOProcessor plugins, the actual output is generated
    by the plugin itself. Hence, we use a generic type for the response data
    """


1626
1627
1628
PoolingRequest: TypeAlias = (
    PoolingCompletionRequest | PoolingChatRequest | IOProcessorRequest
)
1629

1630

1631
class ScoreRequest(OpenAIBaseModel):
1632
1633
1634
1635
    model: str | None = None
    text_1: list[str] | str | ScoreMultiModalParam
    text_2: list[str] | str | ScoreMultiModalParam
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1636

1637
    # --8<-- [start:score-extra-params]
1638

1639
    mm_processor_kwargs: dict[str, Any] | None = Field(
1640
1641
1642
1643
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )

1644
1645
1646
1647
1648
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1649
1650
            "if the served model does not use priority scheduling."
        ),
1651
    )
1652

1653
1654
1655
1656
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )
1657

1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )

    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "Default is True.",
    )
1668
    # --8<-- [end:score-extra-params]
1669

1670
    def to_pooling_params(self):
1671
1672
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1673
            use_activation=get_use_activation(self),
1674
        )
1675
1676


1677
class RerankRequest(OpenAIBaseModel):
1678
1679
1680
    model: str | None = None
    query: str | ScoreMultiModalParam
    documents: list[str] | ScoreMultiModalParam
1681
    top_n: int = Field(default_factory=lambda: 0)
1682
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1683

1684
    # --8<-- [start:rerank-extra-params]
1685

1686
    mm_processor_kwargs: dict[str, Any] | None = Field(
1687
1688
1689
1690
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )

1691
1692
1693
1694
1695
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1696
1697
            "if the served model does not use priority scheduling."
        ),
1698
    )
1699

1700
1701
1702
1703
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )
1704

1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )

    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "Default is True.",
    )
1715
    # --8<-- [end:rerank-extra-params]
1716

1717
    def to_pooling_params(self):
1718
1719
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1720
            use_activation=get_use_activation(self),
1721
        )
1722
1723
1724


class RerankDocument(BaseModel):
1725
1726
    text: str | None = None
    multi_modal: ScoreContentPartParam | None = None
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742


class RerankResult(BaseModel):
    index: int
    document: RerankDocument
    relevance_score: float


class RerankUsage(BaseModel):
    total_tokens: int


class RerankResponse(OpenAIBaseModel):
    id: str
    model: str
    usage: RerankUsage
1743
    results: list[RerankResult]
1744
1745


1746
class CompletionLogProbs(OpenAIBaseModel):
1747
    text_offset: list[int] = Field(default_factory=list)
1748
    token_logprobs: list[float | None] = Field(default_factory=list)
1749
    tokens: list[str] = Field(default_factory=list)
1750
    top_logprobs: list[dict[str, float] | None] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
1751
1752


1753
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1754
1755
    index: int
    text: str
1756
1757
1758
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1759
1760
1761
1762
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1763
1764
            "including encountering the EOS token"
        ),
1765
    )
1766
1767
1768
    token_ids: list[int] | None = None  # For response
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None  # For prompt
Zhuohan Li's avatar
Zhuohan Li committed
1769
1770


1771
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1772
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
1773
    object: Literal["text_completion"] = "text_completion"
Zhuohan Li's avatar
Zhuohan Li committed
1774
1775
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1776
    choices: list[CompletionResponseChoice]
1777
1778
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1779
    usage: UsageInfo
1780
1781

    # vLLM-specific fields that are not in OpenAI spec
1782
    kv_transfer_params: dict[str, Any] | None = Field(
1783
1784
        default=None, description="KVTransfer parameters."
    )
Zhuohan Li's avatar
Zhuohan Li committed
1785
1786


1787
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1788
1789
    index: int
    text: str
1790
1791
1792
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1793
1794
1795
1796
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1797
1798
            "including encountering the EOS token"
        ),
1799
    )
1800
1801
    # not part of the OpenAI spec but for tracing the tokens
    # prompt tokens is put into choice to align with CompletionResponseChoice
1802
1803
    prompt_token_ids: list[int] | None = None
    token_ids: list[int] | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1804
1805


1806
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1807
1808
1809
1810
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1811
    choices: list[CompletionResponseStreamChoice]
1812
    usage: UsageInfo | None = Field(default=None)
1813
1814


1815
class EmbeddingResponseData(OpenAIBaseModel):
1816
1817
    index: int
    object: str = "embedding"
1818
    embedding: list[float] | str
1819
1820


1821
class EmbeddingResponse(OpenAIBaseModel):
1822
    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
1823
1824
1825
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1826
    data: list[EmbeddingResponseData]
1827
1828
1829
    usage: UsageInfo


1830
1831
1832
1833
1834
1835
class EmbeddingBytesResponse(OpenAIBaseModel):
    body: list[bytes]
    metadata: str
    media_type: str = "application/octet-stream"


1836
1837
1838
class PoolingResponseData(OpenAIBaseModel):
    index: int
    object: str = "pooling"
1839
    data: list[list[float]] | list[float] | str
1840
1841
1842
1843
1844
1845
1846


class PoolingResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"pool-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1847
    data: list[PoolingResponseData]
1848
1849
1850
    usage: UsageInfo


1851
1852
1853
1854
1855
1856
class PoolingBytesResponse(OpenAIBaseModel):
    body: list[bytes]
    metadata: str
    media_type: str = "application/octet-stream"


1857
1858
1859
class ScoreResponseData(OpenAIBaseModel):
    index: int
    object: str = "score"
1860
    score: float
1861
1862
1863
1864
1865
1866
1867


class ScoreResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1868
    data: list[ScoreResponseData]
1869
1870
1871
    usage: UsageInfo


1872
class ClassificationCompletionRequest(OpenAIBaseModel):
1873
1874
    model: str | None = None
    input: list[str] | str
1875
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
1876
    user: str | None = None
1877

1878
    # --8<-- [start:classification-extra-params]
1879
1880
1881
1882
1883
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1884
1885
            "if the served model does not use priority scheduling."
        ),
1886
    )
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."
        ),
    )
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
1902
1903
1904
1905
1906
1907
1908
1909
1910
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )

    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )
1911

1912
1913
1914
1915
1916
    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "Default is True.",
    )
1917
    # --8<-- [end:classification-extra-params]
1918
1919

    def to_pooling_params(self):
1920
1921
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1922
            use_activation=get_use_activation(self),
1923
        )
1924
1925


1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
class ClassificationChatRequest(OpenAIBaseModel):
    model: str | None = None
    messages: list[ChatCompletionMessageParam]
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    user: str | None = None

    # --8<-- [start:chat-classification-extra-params]
    add_generation_prompt: bool = Field(
        default=False,
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
    )

    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
            "default)."
        ),
    )

    chat_template: str | None = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."
        ),
    )

    chat_template_kwargs: dict[str, Any] | None = Field(
        default=None,
        description=(
            "Additional keyword args to pass to the template renderer. "
            "Will be accessible by the chat template."
        ),
    )

    mm_processor_kwargs: dict[str, Any] | None = Field(
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )

    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."
        ),
    )

    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    softmax: bool | None = Field(
        default=None,
        description="softmax will be deprecated, please use use_activation instead.",
    )

    activation: bool | None = Field(
        default=None,
        description="activation will be deprecated, please use use_activation instead.",
    )

    use_activation: bool | None = Field(
        default=None,
        description="Whether to use activation for classification outputs. "
        "Default is True.",
    )
    # --8<-- [end:chat-classification-extra-params]

    def to_pooling_params(self):
        return PoolingParams(
            truncate_prompt_tokens=self.truncate_prompt_tokens,
            use_activation=get_use_activation(self),
        )


ClassificationRequest: TypeAlias = (
    ClassificationCompletionRequest | ClassificationChatRequest
)


2022
2023
class ClassificationData(OpenAIBaseModel):
    index: int
2024
    label: str | None
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
    probs: list[float]
    num_classes: int


class ClassificationResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"classify-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    data: list[ClassificationData]
    usage: UsageInfo


2038
2039
2040
2041
2042
2043
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
2044
    id: str = Field(default_factory=make_tool_call_id)
2045
2046
2047
2048
    type: Literal["function"] = "function"
    function: FunctionCall


2049
class DeltaFunctionCall(BaseModel):
2050
2051
    name: str | None = None
    arguments: str | None = None
2052
2053
2054
2055


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
2056
2057
    id: str | None = None
    type: Literal["function"] | None = None
2058
    index: int
2059
    function: DeltaFunctionCall | None = None
2060
2061
2062
2063
2064
2065
2066


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
2067
    tool_calls: list[ToolCall]
2068
2069
2070

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
2071
    content: str | None = None
2072
2073


2074
class ChatMessage(OpenAIBaseModel):
2075
    role: str
2076
2077
2078
2079
2080
    content: str | None = None
    refusal: str | None = None
    annotations: OpenAIAnnotation | None = None
    audio: OpenAIChatCompletionAudio | None = None
    function_call: FunctionCall | None = None
2081
    tool_calls: list[ToolCall] = Field(default_factory=list)
2082

2083
    # vLLM-specific fields that are not in OpenAI spec
2084
    reasoning: str | None = None
2085
    reasoning_content: str | None = None
2086
2087
2088
2089
2090
2091
2092
    """Deprecated: use `reasoning` instead."""

    @model_validator(mode="after")
    def handle_deprecated_reasoning_content(self):
        """Copy reasoning to reasoning_content for backward compatibility."""
        self.reasoning_content = self.reasoning
        return self
2093

2094

2095
2096
2097
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
2098
    bytes: list[int] | None = None
2099
2100
2101


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
2102
2103
    # Workaround: redefine fields name cache so that it's not
    # shared with the super class.
2104
    field_names: ClassVar[set[str] | None] = None
2105
    top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
2106
2107
2108


class ChatCompletionLogProbs(OpenAIBaseModel):
2109
    content: list[ChatCompletionLogProbsContent] | None = None
2110
2111


2112
class ChatCompletionResponseChoice(OpenAIBaseModel):
2113
2114
    index: int
    message: ChatMessage
2115
    logprobs: ChatCompletionLogProbs | None = None
2116
    # per OpenAI spec this is the default
2117
    finish_reason: str | None = "stop"
2118
    # not part of the OpenAI spec but included in vLLM for legacy reasons
2119
    stop_reason: int | str | None = None
2120
2121
    # not part of the OpenAI spec but is useful for tracing the tokens
    # in agent scenarios
2122
    token_ids: list[int] | None = None
2123
2124


2125
class ChatCompletionResponse(OpenAIBaseModel):
2126
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
2127
    object: Literal["chat.completion"] = "chat.completion"
2128
2129
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
2130
    choices: list[ChatCompletionResponseChoice]
2131
2132
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
2133
    usage: UsageInfo
2134
2135

    # vLLM-specific fields that are not in OpenAI spec
2136
2137
2138
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None
    kv_transfer_params: dict[str, Any] | None = Field(
2139
2140
        default=None, description="KVTransfer parameters."
    )
2141
2142


2143
class DeltaMessage(OpenAIBaseModel):
2144
2145
    role: str | None = None
    content: str | None = None
2146
    reasoning: str | None = None
2147
    reasoning_content: str | None = None
2148
    """Deprecated: use `reasoning` instead."""
2149
    tool_calls: list[DeltaToolCall] = Field(default_factory=list)
2150

2151
2152
2153
2154
2155
2156
    @model_validator(mode="after")
    def handle_deprecated_reasoning_content(self):
        """Copy reasoning to reasoning_content for backward compatibility."""
        self.reasoning_content = self.reasoning
        return self

2157

2158
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
2159
2160
    index: int
    delta: DeltaMessage
2161
2162
2163
    logprobs: ChatCompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2164
    # not part of the OpenAI spec but for tracing the tokens
2165
    token_ids: list[int] | None = None
2166
2167


2168
class ChatCompletionStreamResponse(OpenAIBaseModel):
2169
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
2170
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
2171
2172
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
2173
    choices: list[ChatCompletionResponseStreamChoice]
2174
    usage: UsageInfo | None = Field(default=None)
2175
    # not part of the OpenAI spec but for tracing the tokens
2176
    prompt_token_ids: list[int] | None = None
2177
2178


2179
2180
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
2181
2182
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2183
2184
2185
2186
2187
2188
2189
2190


class TranscriptionStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
    object: Literal["transcription.chunk"] = "transcription.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranscriptionResponseStreamChoice]
2191
    usage: UsageInfo | None = Field(default=None)
2192
2193


2194
2195
class InputTokensDetails(OpenAIBaseModel):
    cached_tokens: int
2196
2197
    input_tokens_per_turn: list[int] = Field(default_factory=list)
    cached_tokens_per_turn: list[int] = Field(default_factory=list)
2198
2199
2200


class OutputTokensDetails(OpenAIBaseModel):
2201
2202
    reasoning_tokens: int = 0
    tool_output_tokens: int = 0
2203
2204
    output_tokens_per_turn: list[int] = Field(default_factory=list)
    tool_output_tokens_per_turn: list[int] = Field(default_factory=list)
2205
2206
2207
2208
2209
2210
2211
2212


class ResponseUsage(OpenAIBaseModel):
    input_tokens: int
    input_tokens_details: InputTokensDetails
    output_tokens: int
    output_tokens_details: OutputTokensDetails
    total_tokens: int
2213
2214


2215
2216
2217
2218
2219
2220
def serialize_message(msg):
    """
    Serializes a single message
    """
    if isinstance(msg, dict):
        return msg
2221
    elif hasattr(msg, "to_dict"):
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
        return msg.to_dict()
    else:
        # fallback to pyandic dump
        return msg.model_dump_json()


def serialize_messages(msgs):
    """
    Serializes multiple messages
    """
    return [serialize_message(msg) for msg in msgs] if msgs else None


2235
2236
2237
2238
class ResponsesResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
    created_at: int = Field(default_factory=lambda: int(time.time()))
    # error: Optional[ResponseError] = None
2239
2240
2241
    incomplete_details: IncompleteDetails | None = None
    instructions: str | None = None
    metadata: Metadata | None = None
2242
2243
    model: str
    object: Literal["response"] = "response"
2244
    output: list[ResponseOutputItem]
2245
2246
2247
2248
2249
2250
2251
    parallel_tool_calls: bool
    temperature: float
    tool_choice: ToolChoice
    tools: list[Tool]
    top_p: float
    background: bool
    max_output_tokens: int
2252
2253
2254
2255
    max_tool_calls: int | None = None
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
2256
2257
    service_tier: Literal["auto", "default", "flex", "scale", "priority"]
    status: ResponseStatus
2258
2259
    text: ResponseTextConfig | None = None
    top_logprobs: int | None = None
2260
    truncation: Literal["auto", "disabled"]
2261
2262
    usage: ResponseUsage | None = None
    user: str | None = None
2263

2264
2265
2266
2267
    # --8<-- [start:responses-extra-params]
    # These are populated when enable_response_messages is set to True
    # NOTE: custom serialization is needed
    # see serialize_input_messages and serialize_output_messages
2268
2269
    input_messages: list[ChatCompletionMessageParam] | None = None
    output_messages: list[ChatCompletionMessageParam] | None = None
2270
2271
2272
2273
2274
2275
2276
    # --8<-- [end:responses-extra-params]

    # NOTE: openAI harmony doesn't serialize TextContent properly,
    # TODO: this fixes for TextContent, but need to verify for tools etc
    # https://github.com/openai/harmony/issues/78
    @field_serializer("output_messages", when_used="json")
    def serialize_output_messages(self, msgs, _info):
2277
        return serialize_messages(msgs)
2278
2279
2280
2281
2282

    # NOTE: openAI harmony doesn't serialize TextContent properly, this fixes it
    # https://github.com/openai/harmony/issues/78
    @field_serializer("input_messages", when_used="json")
    def serialize_input_messages(self, msgs, _info):
2283
        return serialize_messages(msgs)
2284

2285
2286
2287
2288
2289
2290
2291
2292
2293
    @classmethod
    def from_request(
        cls,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
        model_name: str,
        created_time: int,
        output: list[ResponseOutputItem],
        status: ResponseStatus,
2294
2295
2296
        usage: ResponseUsage | None = None,
        input_messages: list[ChatCompletionMessageParam] | None = None,
        output_messages: list[ChatCompletionMessageParam] | None = None,
2297
    ) -> "ResponsesResponse":
2298
        incomplete_details: IncompleteDetails | None = None
2299
2300
        if status == "incomplete":
            incomplete_details = IncompleteDetails(reason="max_output_tokens")
2301
2302
2303
        # TODO: implement the other reason for incomplete_details,
        # which is content_filter
        # incomplete_details = IncompleteDetails(reason='content_filter')
2304
2305
2306
        return cls(
            id=request.request_id,
            created_at=created_time,
2307
            incomplete_details=incomplete_details,
2308
2309
2310
2311
            instructions=request.instructions,
            metadata=request.metadata,
            model=model_name,
            output=output,
2312
2313
            input_messages=input_messages,
            output_messages=output_messages,
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
            parallel_tool_calls=request.parallel_tool_calls,
            temperature=sampling_params.temperature,
            tool_choice=request.tool_choice,
            tools=request.tools,
            top_p=sampling_params.top_p,
            background=request.background,
            max_output_tokens=sampling_params.max_tokens,
            max_tool_calls=request.max_tool_calls,
            previous_response_id=request.previous_response_id,
            prompt=request.prompt,
            reasoning=request.reasoning,
            service_tier=request.service_tier,
            status=status,
            text=request.text,
            top_logprobs=sampling_params.logprobs,
            truncation=request.truncation,
            user=request.user,
            usage=usage,
        )


2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartDoneEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.done"]
    """The type of the event. Always `response.reasoning_part.done`."""


# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartAddedEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.added"]
    """The type of the event. Always `response.reasoning_part.added`."""


2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
# vLLM Streaming Events
# Note: we override the response type with the vLLM ResponsesResponse type
class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseInProgressEvent(OpenAIResponseInProgressEvent):
    response: ResponsesResponse  # type: ignore[override]


2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
StreamingResponsesResponse: TypeAlias = (
    ResponseCreatedEvent
    | ResponseInProgressEvent
    | ResponseCompletedEvent
    | ResponseOutputItemAddedEvent
    | ResponseOutputItemDoneEvent
    | ResponseContentPartAddedEvent
    | ResponseContentPartDoneEvent
    | ResponseReasoningTextDeltaEvent
    | ResponseReasoningTextDoneEvent
    | ResponseReasoningPartAddedEvent
    | ResponseReasoningPartDoneEvent
    | ResponseCodeInterpreterCallInProgressEvent
    | ResponseCodeInterpreterCallCodeDeltaEvent
    | ResponseWebSearchCallInProgressEvent
    | ResponseWebSearchCallSearchingEvent
    | ResponseWebSearchCallCompletedEvent
    | ResponseCodeInterpreterCallCodeDoneEvent
    | ResponseCodeInterpreterCallInterpretingEvent
    | ResponseCodeInterpreterCallCompletedEvent
)
2414

2415
2416
2417
BatchRequestInputBody: TypeAlias = (
    ChatCompletionRequest | EmbeddingRequest | ScoreRequest | RerankRequest
)
2418
2419


2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

2439
    # The parameters of the request.
2440
    body: BatchRequestInputBody
2441

2442
    @field_validator("body", mode="plain")
2443
2444
2445
    @classmethod
    def check_type_for_url(cls, value: Any, info: ValidationInfo):
        # Use url to disambiguate models
2446
        url: str = info.data["url"]
2447
2448
2449
2450
        if url == "/v1/chat/completions":
            return ChatCompletionRequest.model_validate(value)
        if url == "/v1/embeddings":
            return TypeAdapter(EmbeddingRequest).validate_python(value)
2451
        if url.endswith("/score"):
2452
            return ScoreRequest.model_validate(value)
2453
2454
2455
        if url.endswith("/rerank"):
            return RerankRequest.model_validate(value)
        return TypeAdapter(BatchRequestInputBody).validate_python(value)
2456

2457

2458
2459
2460
2461
2462
2463
2464
2465
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
2466
2467
2468
2469
2470
2471
2472
    body: (
        ChatCompletionResponse
        | EmbeddingResponse
        | ScoreResponse
        | RerankResponse
        | None
    ) = None
2473
2474


2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

2486
    response: BatchResponseData | None
2487
2488
2489

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
2490
    error: Any | None
2491
2492


2493
class TokenizeCompletionRequest(OpenAIBaseModel):
2494
    model: str | None = None
2495
2496
    prompt: str

2497
2498
2499
2500
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
2501
2502
            "the prompt."
        ),
2503
    )
2504
    return_token_strs: bool | None = Field(
2505
        default=False,
2506
2507
2508
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
2509
    )
2510
2511
2512


class TokenizeChatRequest(OpenAIBaseModel):
2513
    model: str | None = None
2514
    messages: list[ChatCompletionMessageParam]
2515

2516
2517
    add_generation_prompt: bool = Field(
        default=True,
2518
2519
2520
2521
2522
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
2523
    )
2524
    return_token_strs: bool | None = Field(
2525
        default=False,
2526
2527
2528
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
2529
    )
2530
2531
    continue_final_message: bool = Field(
        default=False,
2532
2533
2534
2535
2536
2537
2538
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
2539
2540
2541
2542
2543
2544
2545
2546
    )
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
2547
2548
            "default)."
        ),
2549
    )
2550
    chat_template: str | None = Field(
2551
2552
2553
2554
2555
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
2556
2557
            "does not define one."
        ),
2558
    )
2559
    chat_template_kwargs: dict[str, Any] | None = Field(
2560
        default=None,
2561
2562
        description=(
            "Additional keyword args to pass to the template renderer. "
2563
2564
            "Will be accessible by the chat template."
        ),
2565
    )
2566
    mm_processor_kwargs: dict[str, Any] | None = Field(
2567
2568
2569
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
2570
    tools: list[ChatCompletionToolsParam] | None = Field(
2571
2572
2573
        default=None,
        description=("A list of tools the model may call."),
    )
2574

2575
2576
2577
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
2578
2579
2580
2581
2582
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
2583
2584
        return data

2585

2586
TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest
2587
2588
2589
2590
2591


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
2592
    tokens: list[int]
2593
    token_strs: list[str] | None = None
2594
2595
2596


class DetokenizeRequest(OpenAIBaseModel):
2597
    model: str | None = None
2598
    tokens: list[int]
2599
2600
2601
2602


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
2603
2604


2605
2606
class TokenizerInfoResponse(OpenAIBaseModel):
    """
2607
    Response containing tokenizer configuration
2608
2609
2610
2611
2612
2613
2614
    equivalent to tokenizer_config.json
    """

    model_config = ConfigDict(extra="allow")
    tokenizer_class: str


2615
class LoadLoRAAdapterRequest(BaseModel):
2616
2617
2618
2619
    lora_name: str
    lora_path: str


2620
class UnloadLoRAAdapterRequest(BaseModel):
2621
    lora_name: str
2622
    lora_int_id: int | None = Field(default=None)
2623
2624
2625


## Protocols for Audio
2626
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
2627
2628
2629
2630


class TranscriptionRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
2631
    # https://platform.openai.com/docs/api-reference/audio/createTranscription
2632
2633
2634
2635
2636
2637
2638

    file: UploadFile
    """
    The audio file object (not file name) to transcribe, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

2639
    model: str | None = None
2640
2641
2642
    """ID of the model to use.
    """

2643
    language: str | None = None
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
    """The language of the input audio.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy and latency.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    ## TODO (varun) : Support if set to 0, certain thresholds are met !!

2667
    timestamp_granularities: list[Literal["word", "segment"]] = Field(
2668
2669
        alias="timestamp_granularities[]", default=[]
    )
2670
2671
2672
2673
2674
2675
2676
2677
    """The timestamp granularities to populate for this transcription.

    `response_format` must be set `verbose_json` to use timestamp granularities.
    Either or both of these options are supported: `word`, or `segment`. Note:
    There is no additional latency for segment timestamps, but generating word
    timestamps incurs additional latency.
    """

2678
    stream: bool | None = False
2679
    """When set, it will enable output to be streamed in a similar fashion
2680
    as the Chat Completion endpoint.
2681
    """
2682
    # --8<-- [start:transcription-extra-params]
2683
    # Flattened stream option to simplify form data.
2684
2685
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2686

2687
    vllm_xargs: dict[str, str | int | float] | None = Field(
2688
        default=None,
2689
2690
2691
2692
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
2693
    )
2694
    # --8<-- [end:transcription-extra-params]
2695

2696
    to_language: str | None = None
2697
2698
    """The language of the output audio we transcribe to.

2699
    Please note that this is not currently used by supported models at this
2700
2701
2702
    time, but it is a placeholder for future use, matching translation api.
    """

2703
    # --8<-- [start:transcription-sampling-params]
2704
2705
2706
2707
2708
2709
2710
2711
2712
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """

2713
    top_p: float | None = None
2714
    """Enables nucleus (top-p) sampling, where tokens are selected from the
2715
2716
2717
    smallest possible set whose cumulative probability exceeds `p`.
    """

2718
    top_k: int | None = None
2719
2720
    """Limits sampling to the `k` most probable tokens at each step."""

2721
    min_p: float | None = None
2722
    """Filters out tokens with a probability lower than `min_p`, ensuring a
2723
2724
2725
    minimum likelihood threshold during sampling.
    """

2726
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2727
2728
    """The seed to use for sampling."""

2729
    frequency_penalty: float | None = 0.0
2730
2731
    """The frequency penalty to use for sampling."""

2732
    repetition_penalty: float | None = None
2733
2734
    """The repetition penalty to use for sampling."""

2735
    presence_penalty: float | None = 0.0
2736
    """The presence penalty to use for sampling."""
2737
    # --8<-- [end:transcription-sampling-params]
2738

2739
2740
    # Default sampling parameters for transcription requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
2741
2742
2743
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
2744
        "top_k": 0,
2745
        "min_p": 0.0,
2746
2747
2748
    }

    def to_sampling_params(
2749
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2750
    ) -> SamplingParams:
2751
2752
2753
2754
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
2755

2756
2757
2758
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2759
2760
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2761
2762
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
2763
2764
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
2765
2766
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
2767
2768
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
2769
2770
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
2771
2772
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
2773
2774
2775
2776

        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )

        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=repetition_penalty,
            presence_penalty=self.presence_penalty,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
            extra_args=self.vllm_xargs,
        )
2795
2796
2797

    @model_validator(mode="before")
    @classmethod
2798
2799
2800
2801
2802
2803
2804
    def validate_transcription_request(cls, data):
        if isinstance(data.get("file"), str):
            raise HTTPException(
                status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
                detail="Expected 'file' to be a file-like object, not 'str'.",
            )

2805
2806
2807
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
2808
            raise ValueError("Stream options can only be defined when `stream=True`.")
2809
2810

        return data
2811
2812
2813


# Transcription response objects
2814
2815
2816
2817
2818
class TranscriptionUsageAudio(OpenAIBaseModel):
    type: Literal["duration"] = "duration"
    seconds: int


2819
2820
2821
class TranscriptionResponse(OpenAIBaseModel):
    text: str
    """The transcribed text."""
2822
    usage: TranscriptionUsageAudio
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873


class TranscriptionWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranscriptionSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

    avg_logprob: float
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

    compression_ratio: float
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

    no_speech_prob: float
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

2874
    tokens: list[int]
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
    """Array of token IDs for the text content."""


class TranscriptionResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The transcribed text."""

2888
    segments: list[TranscriptionSegment] | None = None
2889
2890
    """Segments of the transcribed text and their corresponding details."""

2891
    words: list[TranscriptionWord] | None = None
2892
    """Extracted words and their corresponding timestamps."""
2893
2894
2895
2896


class TranslationResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
2897
2898
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2899
2900
2901
2902
2903
2904
2905
2906


class TranslationStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
    object: Literal["translation.chunk"] = "translation.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranslationResponseStreamChoice]
2907
    usage: UsageInfo | None = Field(default=None)
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919


class TranslationRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/audio/createTranslation

    file: UploadFile
    """
    The audio file object (not file name) to translate, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

2920
    model: str | None = None
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
    """ID of the model to use.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    # TODO support additional sampling parameters
    # --8<-- [start:translation-sampling-params]
2940
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2941
2942
    """The seed to use for sampling."""

2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """
    # --8<-- [end:translation-sampling-params]

    # --8<-- [start:translation-extra-params]
2954
    language: str | None = None
2955
2956
2957
2958
2959
2960
2961
    """The language of the input audio we translate from.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy.
    """

2962
    to_language: str | None = None
2963
2964
2965
2966
2967
2968
2969
    """The language of the input audio we translate to.

    Please note that this is not supported by all models, refer to the specific
    model documentation for more details.
    For instance, Whisper only supports `to_language=en`.
    """

2970
    stream: bool | None = False
2971
    """Custom field not present in the original OpenAI definition. When set,
2972
    it will enable output to be streamed in a similar fashion as the Chat
2973
    Completion endpoint.
2974
2975
    """
    # Flattened stream option to simplify form data.
2976
2977
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2978
2979
2980
2981
2982
2983
2984
2985
    # --8<-- [end:translation-extra-params]

    # Default sampling parameters for translation requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "temperature": 0,
    }

    def to_sampling_params(
2986
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2987
    ) -> SamplingParams:
2988
2989
2990
2991
2992
2993
2994
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2995
2996
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2997

2998
2999
3000
3001
3002
3003
3004
3005
        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
        )
3006
3007
3008
3009
3010
3011
3012

    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
3013
            raise ValueError("Stream options can only be defined when `stream=True`.")
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086

        return data


# Translation response objects
class TranslationResponse(OpenAIBaseModel):
    text: str
    """The translated text."""


class TranslationWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranslationSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

    avg_logprob: float
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

    compression_ratio: float
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

    no_speech_prob: float
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

    tokens: list[int]
    """Array of token IDs for the text content."""


class TranslationResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The translated text."""

3087
    segments: list[TranslationSegment] | None = None
3088
3089
    """Segments of the translated text and their corresponding details."""

3090
    words: list[TranslationWord] | None = None
3091
    """Extracted words and their corresponding timestamps."""
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168


####### Tokens IN <> Tokens OUT #######
class GenerateRequest(BaseModel):
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    token_ids: list[int]
    """The token ids to generate text from."""

    # features: MultiModalFeatureSpec
    # TODO (NickLucche): implement once Renderer work is completed
    features: str | None = None
    """The processed MM inputs for the model."""

    sampling_params: SamplingParams
    """The sampling parameters for the model."""

    model: str | None = None

    stream: bool | None = False
    stream_options: StreamOptions | None = None
    cache_salt: str | None = Field(
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
            "to 256 bit)."
        ),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."
        ),
    )
    kv_transfer_params: dict[str, Any] | None = Field(
        default=None,
        description="KVTransfer parameters used for disaggregated serving.",
    )


class GenerateResponseChoice(BaseModel):
    index: int
    logprobs: ChatCompletionLogProbs | None = None
    # per OpenAI spec this is the default
    finish_reason: str | None = "stop"
    token_ids: list[int] | None = None


class GenerateResponse(BaseModel):
    request_id: str = Field(
        default_factory=lambda: f"{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    choices: list[GenerateResponseChoice]

    prompt_logprobs: list[dict[int, Logprob] | None] | None = None

    kv_transfer_params: dict[str, Any] | None = Field(
        default=None,
        description="KVTransfer parameters used for disaggregated serving.",
    )