protocol.py 87.2 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import json
Zhuohan Li's avatar
Zhuohan Li committed
7
import time
8
from http import HTTPStatus
9
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
Zhuohan Li's avatar
Zhuohan Li committed
10

11
import regex as re
12
import torch
13
from fastapi import HTTPException, UploadFile
14
from openai.types.chat.chat_completion_audio import (
15
16
17
    ChatCompletionAudio as OpenAIChatCompletionAudio,
)
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
18
19
20
21
22
from openai.types.responses import (
    ResponseCodeInterpreterCallCodeDeltaEvent,
    ResponseCodeInterpreterCallCodeDoneEvent,
    ResponseCodeInterpreterCallCompletedEvent,
    ResponseCodeInterpreterCallInProgressEvent,
23
24
25
26
27
    ResponseCodeInterpreterCallInterpretingEvent,
    ResponseContentPartAddedEvent,
    ResponseContentPartDoneEvent,
    ResponseFunctionToolCall,
    ResponseInputItemParam,
28
29
30
31
    ResponseMcpCallArgumentsDeltaEvent,
    ResponseMcpCallArgumentsDoneEvent,
    ResponseMcpCallCompletedEvent,
    ResponseMcpCallInProgressEvent,
32
33
34
35
36
37
38
39
40
41
42
    ResponseOutputItem,
    ResponseOutputItemAddedEvent,
    ResponseOutputItemDoneEvent,
    ResponsePrompt,
    ResponseReasoningTextDeltaEvent,
    ResponseReasoningTextDoneEvent,
    ResponseStatus,
    ResponseWebSearchCallCompletedEvent,
    ResponseWebSearchCallInProgressEvent,
    ResponseWebSearchCallSearchingEvent,
)
43
from openai.types.responses import (
44
45
46
    ResponseCompletedEvent as OpenAIResponseCompletedEvent,
)
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
47
from openai.types.responses import (
48
49
    ResponseInProgressEvent as OpenAIResponseInProgressEvent,
)
50
from openai.types.responses.response_reasoning_item import (
51
52
    Content as ResponseReasoningTextContent,
)
53
from openai_harmony import Message as OpenAIHarmonyMessage
54
55
56
57
58

# Backward compatibility for OpenAI client versions
try:  # For older openai versions (< 1.100.0)
    from openai.types.responses import ResponseTextConfig
except ImportError:  # For newer openai versions (>= 1.100.0)
59
    from openai.types.responses import ResponseFormatTextConfig as ResponseTextConfig
60

61

62
from openai.types.responses.response import IncompleteDetails, ToolChoice
63
64
from openai.types.responses.tool import Tool
from openai.types.shared import Metadata, Reasoning
65
66
67
68
from pydantic import (
    BaseModel,
    ConfigDict,
    Field,
69
    ValidationError,
70
    field_serializer,
71
72
    model_validator,
)
Zhuohan Li's avatar
Zhuohan Li committed
73

74
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam, make_tool_call_id
75
from vllm.logger import init_logger
76
from vllm.logprobs import Logprob
77
78
79
80
81
82
from vllm.sampling_params import (
    BeamSearchParams,
    RequestOutputKind,
    SamplingParams,
    StructuredOutputsParams,
)
83
84
from vllm.utils import random_uuid
from vllm.utils.import_utils import resolve_obj_by_qualname
85

86
87
logger = init_logger(__name__)

88
_LONG_INFO = torch.iinfo(torch.long)
89

Zhuohan Li's avatar
Zhuohan Li committed
90

91
class OpenAIBaseModel(BaseModel):
92
93
94
    # OpenAI API does allow extra fields
    model_config = ConfigDict(extra="allow")

95
    # Cache class field names
96
    field_names: ClassVar[set[str] | None] = None
97

98
    @model_validator(mode="wrap")
99
    @classmethod
100
101
102
103
    def __log_extra_fields__(cls, data, handler):
        result = handler(data)
        if not isinstance(data, dict):
            return result
104
105
        field_names = cls.field_names
        if field_names is None:
106
107
108
109
            # Get all class field names and their potential aliases
            field_names = set()
            for field_name, field in cls.model_fields.items():
                field_names.add(field_name)
110
                if alias := getattr(field, "alias", None):
111
112
113
114
115
116
                    field_names.add(alias)
            cls.field_names = field_names

        # Compare against both field names and aliases
        if any(k not in field_names for k in data):
            logger.warning(
117
                "The following fields were present in the request but ignored: %s",
118
119
                data.keys() - field_names,
            )
120
        return result
121
122


123
class ErrorInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
124
125
    message: str
    type: str
126
    param: str | None = None
127
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
128
129


130
131
132
133
class ErrorResponse(OpenAIBaseModel):
    error: ErrorInfo


134
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
135
136
137
138
139
140
141
142
143
144
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
145
    group: str | None = None
146
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
147
148


149
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
150
151
152
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
153
    owned_by: str = "vllm"
154
155
156
    root: str | None = None
    parent: str | None = None
    max_model_len: int | None = None
157
    permission: list[ModelPermission] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
158
159


160
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
161
    object: str = "list"
162
    data: list[ModelCard] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
163
164


165
class PromptTokenUsageInfo(OpenAIBaseModel):
166
    cached_tokens: int | None = None
167
168


169
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
170
171
    prompt_tokens: int = 0
    total_tokens: int = 0
172
173
    completion_tokens: int | None = 0
    prompt_tokens_details: PromptTokenUsageInfo | None = None
Zhuohan Li's avatar
Zhuohan Li committed
174
175


176
177
class RequestResponseMetadata(BaseModel):
    request_id: str
178
    final_usage_info: UsageInfo | None = None
179
180


181
182
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
183
    description: str | None = None
184
185
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
186
187
    json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
    strict: bool | None = None
188
189


190
class LegacyStructuralTag(OpenAIBaseModel):
191
192
193
    begin: str
    # schema is the field, but that causes conflicts with pydantic so
    # instead use structural_tag_schema with an alias
194
    structural_tag_schema: dict[str, Any] | None = Field(default=None, alias="schema")
195
196
197
    end: str


198
class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
199
    type: Literal["structural_tag"]
200
    structures: list[LegacyStructuralTag]
201
202
203
    triggers: list[str]


204
205
206
207
208
209
210
211
212
213
class StructuralTagResponseFormat(OpenAIBaseModel):
    type: Literal["structural_tag"]
    format: Any


AnyStructuralTagResponseFormat: TypeAlias = (
    LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
)


214
class ResponseFormat(OpenAIBaseModel):
215
    # type must be "json_schema", "json_object", or "text"
216
    type: Literal["text", "json_object", "json_schema"]
217
    json_schema: JsonSchemaResponseFormat | None = None
218
219


220
221
222
AnyResponseFormat: TypeAlias = (
    ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
)
223
224


225
class StreamOptions(OpenAIBaseModel):
226
227
    include_usage: bool | None = True
    continuous_usage_stats: bool | None = False
228
229


230
231
class FunctionDefinition(OpenAIBaseModel):
    name: str
232
233
    description: str | None = None
    parameters: dict[str, Any] | None = None
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


250
251
# extra="forbid" is a workaround to have kwargs as a field,
# see https://github.com/pydantic/pydantic/issues/3125
252
253
class LogitsProcessorConstructor(BaseModel):
    qualname: str
254
255
    args: list[Any] | None = None
    kwargs: dict[str, Any] | None = None
256

257
258
    model_config = ConfigDict(extra="forbid")

259

260
LogitsProcessors = list[str | LogitsProcessorConstructor]
261
262


263
def get_logits_processors(
264
265
    processors: LogitsProcessors | None, pattern: str | None
) -> list[Any] | None:
266
267
268
    if processors and pattern:
        logits_processors = []
        for processor in processors:
269
            qualname = processor if isinstance(processor, str) else processor.qualname
270
271
272
273
            if not re.match(pattern, qualname):
                raise ValueError(
                    f"Logits processor '{qualname}' is not allowed by this "
                    "server. See --logits-processor-pattern engine argument "
274
275
                    "for more information."
                )
276
277
278
279
280
281
282
            try:
                logits_processor = resolve_obj_by_qualname(qualname)
            except Exception as e:
                raise ValueError(
                    f"Logits processor '{qualname}' could not be resolved: {e}"
                ) from e
            if isinstance(processor, LogitsProcessorConstructor):
283
284
285
                logits_processor = logits_processor(
                    *processor.args or [], **processor.kwargs or {}
                )
286
287
288
289
290
            logits_processors.append(logits_processor)
        return logits_processors
    elif processors:
        raise ValueError(
            "The `logits_processors` argument is not supported by this "
291
            "server. See --logits-processor-pattern engine argument "
292
293
            "for more information."
        )
294
295
296
    return None


297
ResponseInputOutputItem: TypeAlias = ResponseInputItemParam | ResponseOutputItem
298
299


300
301
302
class ResponsesRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/responses/create
303
304
    background: bool | None = False
    include: (
305
306
307
308
309
310
311
312
313
314
        list[
            Literal[
                "code_interpreter_call.outputs",
                "computer_call_output.output.image_url",
                "file_search_call.results",
                "message.input_image.image_url",
                "message.output_text.logprobs",
                "reasoning.encrypted_content",
            ],
        ]
315
316
317
318
319
320
321
322
        | None
    ) = None
    input: str | list[ResponseInputOutputItem]
    instructions: str | None = None
    max_output_tokens: int | None = None
    max_tool_calls: int | None = None
    metadata: Metadata | None = None
    model: str | None = None
323
    logit_bias: dict[str, float] | None = None
324
325
326
327
    parallel_tool_calls: bool | None = True
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
328
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto"
329
330
331
332
    store: bool | None = True
    stream: bool | None = False
    temperature: float | None = None
    text: ResponseTextConfig | None = None
333
334
    tool_choice: ToolChoice = "auto"
    tools: list[Tool] = Field(default_factory=list)
335
336
    top_logprobs: int | None = 0
    top_p: float | None = None
337
    top_k: int | None = None
338
339
    truncation: Literal["auto", "disabled"] | None = "disabled"
    user: str | None = None
340
341
342
343
344
345
346

    # --8<-- [start:responses-extra-params]
    request_id: str = Field(
        default_factory=lambda: f"resp_{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
347
348
            "through out the inference process and return in response."
        ),
349
    )
350
    mm_processor_kwargs: dict[str, Any] | None = Field(
351
352
353
354
355
356
357
358
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
359
360
            "if the served model does not use priority scheduling."
        ),
361
    )
362
    cache_salt: str | None = Field(
363
364
365
366
367
368
369
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
370
            "to 256 bit)."
371
372
        ),
    )
373
374
375
376
377

    enable_response_messages: bool = Field(
        default=False,
        description=(
            "Dictates whether or not to return messages as part of the "
378
            "response object. Currently only supported for"
379
380
381
            "non-background and gpt-oss only. "
        ),
    )
382
383
384
385
386
    # similar to input_messages / output_messages in ResponsesResponse
    # we take in previous_input_messages (ie in harmony format)
    # this cannot be used in conjunction with previous_response_id
    # TODO: consider supporting non harmony messages as well
    previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None
387
388
389
390
391
    # --8<-- [end:responses-extra-params]

    _DEFAULT_SAMPLING_PARAMS = {
        "temperature": 1.0,
        "top_p": 1.0,
392
        "top_k": 0,
393
394
395
396
397
    }

    def to_sampling_params(
        self,
        default_max_tokens: int,
398
        default_sampling_params: dict | None = None,
399
400
401
402
403
404
405
406
407
    ) -> SamplingParams:
        if self.max_output_tokens is None:
            max_tokens = default_max_tokens
        else:
            max_tokens = min(self.max_output_tokens, default_max_tokens)

        default_sampling_params = default_sampling_params or {}
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
408
409
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
410
411
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
412
413
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
414
415
416
417
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
418
        stop_token_ids = default_sampling_params.get("stop_token_ids")
419
420

        # Structured output
421
        structured_outputs = None
422
423
        if self.text is not None and self.text.format is not None:
            response_format = self.text.format
424
425
426
427
            if (
                response_format.type == "json_schema"
                and response_format.schema_ is not None
            ):
428
                structured_outputs = StructuredOutputsParams(
429
430
                    json=response_format.schema_
                )
431
432
433
434
435
436
437
            elif response_format.type == "json_object":
                raise NotImplementedError("json_object is not supported")

        # TODO: add more parameters
        return SamplingParams.from_optional(
            temperature=temperature,
            top_p=top_p,
438
            top_k=top_k,
439
            max_tokens=max_tokens,
440
            logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
441
            stop_token_ids=stop_token_ids,
442
443
444
            output_kind=(
                RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
            ),
445
            structured_outputs=structured_outputs,
446
            logit_bias=self.logit_bias,
447
448
        )

449
450
451
452
    def is_include_output_logprobs(self) -> bool:
        """Check if the request includes output logprobs."""
        if self.include is None:
            return False
453
454
455
456
        return (
            isinstance(self.include, list)
            and "message.output_text.logprobs" in self.include
        )
457

458
459
460
461
462
    @model_validator(mode="before")
    def validate_background(cls, data):
        if not data.get("background"):
            return data
        if not data.get("store", True):
463
            raise ValueError("background can only be used when `store` is true")
464
465
466
467
468
469
470
471
        return data

    @model_validator(mode="before")
    def validate_prompt(cls, data):
        if data.get("prompt") is not None:
            raise ValueError("prompt template is not supported")
        return data

472
473
    @model_validator(mode="before")
    def check_cache_salt_support(cls, data):
474
475
476
477
478
479
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
480
481
        return data

482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
    @model_validator(mode="before")
    def function_call_parsing(cls, data):
        """Parse function_call dictionaries into ResponseFunctionToolCall objects.
        This ensures Pydantic can properly resolve union types in the input field.
        Function calls provided as dicts are converted to ResponseFunctionToolCall
        objects before validation, while invalid structures are left for Pydantic
        to reject with appropriate error messages.
        """

        input_data = data.get("input")

        # Early return for None, strings, or bytes
        # (strings are iterable but shouldn't be processed)
        if input_data is None or isinstance(input_data, (str, bytes)):
            return data

        # Convert iterators (like ValidatorIterator) to list
        if not isinstance(input_data, list):
            try:
                input_data = list(input_data)
            except TypeError:
                # Not iterable, leave as-is for Pydantic to handle
                return data

        processed_input = []
        for item in input_data:
            if isinstance(item, dict) and item.get("type") == "function_call":
                try:
                    processed_input.append(ResponseFunctionToolCall(**item))
                except ValidationError:
                    # Let Pydantic handle validation for malformed function calls
                    logger.debug(
                        "Failed to parse function_call to ResponseFunctionToolCall, "
                        "leaving for Pydantic validation"
                    )
                    processed_input.append(item)
            else:
                processed_input.append(item)

        data["input"] = processed_input
        return data

524

525
class ChatCompletionRequest(OpenAIBaseModel):
526
527
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
528
    messages: list[ChatCompletionMessageParam]
529
530
531
532
533
534
    model: str | None = None
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: bool | None = False
    top_logprobs: int | None = 0
    max_tokens: int | None = Field(
535
        default=None,
536
537
        deprecated="max_tokens is deprecated in favor of "
        "the max_completion_tokens field",
538
    )
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
    max_completion_tokens: int | None = None
    n: int | None = 1
    presence_penalty: float | None = 0.0
    response_format: AnyResponseFormat | None = None
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    temperature: float | None = None
    top_p: float | None = None
    tools: list[ChatCompletionToolsParam] | None = None
    tool_choice: (
        Literal["none"]
        | Literal["auto"]
        | Literal["required"]
        | ChatCompletionNamedToolChoiceParam
        | None
    ) = "none"
    reasoning_effort: Literal["low", "medium", "high"] | None = None
558
    include_reasoning: bool = True
559
    parallel_tool_calls: bool | None = True
560

561
    # NOTE this will be ignored by vLLM
562
    user: str | None = None
563

564
    # --8<-- [start:chat-completion-sampling-params]
565
    use_beam_search: bool = False
566
567
568
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
569
    length_penalty: float = 1.0
570
    stop_token_ids: list[int] | None = []
571
572
573
574
575
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
576
577
578
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    prompt_logprobs: int | None = None
    allowed_token_ids: list[int] | None = None
579
    bad_words: list[str] = Field(default_factory=list)
580
    # --8<-- [end:chat-completion-sampling-params]
581

582
    # --8<-- [start:chat-completion-extra-params]
583
    echo: bool = Field(
584
585
586
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
587
588
            "if they belong to the same role."
        ),
589
    )
590
    add_generation_prompt: bool = Field(
591
        default=True,
592
593
594
595
596
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
597
    )
598
599
    continue_final_message: bool = Field(
        default=False,
600
601
602
603
604
605
606
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
607
    )
608
    add_special_tokens: bool = Field(
609
610
611
612
613
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
614
            "special tokens so this should be set to false (as is the "
615
616
            "default)."
        ),
617
    )
618
    documents: list[dict[str, str]] | None = Field(
619
        default=None,
620
621
622
623
624
625
626
        description=(
            "A list of dicts representing documents that will be accessible to "
            "the model if it is performing RAG (retrieval-augmented generation)."
            " If the template does not support RAG, this argument will have no "
            "effect. We recommend that each document should be a dict containing "
            '"title" and "text" keys.'
        ),
627
    )
628
    chat_template: str | None = Field(
629
630
631
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
632
633
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
634
635
            "does not define one."
        ),
636
    )
637
    chat_template_kwargs: dict[str, Any] | None = Field(
638
        default=None,
639
640
        description=(
            "Additional keyword args to pass to the template renderer. "
641
642
            "Will be accessible by the chat template."
        ),
643
    )
644
    mm_processor_kwargs: dict[str, Any] | None = Field(
645
646
647
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
648
    structured_outputs: StructuredOutputsParams | None = Field(
649
        default=None,
650
        description="Additional kwargs for structured outputs",
651
    )
652
653
654
655
656
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
657
658
            "if the served model does not use priority scheduling."
        ),
659
    )
660
    request_id: str = Field(
661
        default_factory=random_uuid,
662
663
664
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
665
666
            "through out the inference process and return in response."
        ),
667
    )
668
    logits_processors: LogitsProcessors | None = Field(
669
670
671
672
673
674
675
676
677
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
678
679
680
            "{'param': 'value'}}."
        ),
    )
681
    return_tokens_as_token_ids: bool | None = Field(
682
683
684
685
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
686
687
688
            "that are not JSON-encodable can be identified."
        ),
    )
689
    return_token_ids: bool | None = Field(
690
691
692
693
694
695
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
696
697
698
            "need to map generated text back to input tokens."
        ),
    )
699
    cache_salt: str | None = Field(
700
701
702
703
704
705
706
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
707
            "to 256 bit)."
708
709
        ),
    )
710
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
711
        default=None,
712
713
        description="KVTransfer parameters used for disaggregated serving.",
    )
714

715
    vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
716
        default=None,
717
        description=(
718
            "Additional request parameters with (list of) string or "
719
720
            "numeric values, used by custom extensions."
        ),
721
722
    )

723
    # --8<-- [end:chat-completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
724

725
726
727
728
729
    # Default sampling parameters for chat completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
730
        "top_k": 0,
731
732
733
734
        "min_p": 0.0,
    }

    def to_beam_search_params(
735
736
        self, max_tokens: int, default_sampling_params: dict
    ) -> BeamSearchParams:
737
        n = self.n if self.n is not None else 1
738
739
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
740
741
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
742
743
744
745
746
747

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
748
            length_penalty=self.length_penalty,
749
750
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
751

752
    def to_sampling_params(
753
        self,
754
        max_tokens: int,
755
        logits_processor_pattern: str | None,
756
        default_sampling_params: dict,
757
    ) -> SamplingParams:
758
759
760
761
762
763
764
765
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
766
767
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
768
769
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
770
771
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
772
773
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
774
775
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
776
777
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
778
779
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
780

781
782
783
784
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

785
        response_format = self.response_format
786
        if response_format is not None:
787
788
789
790
791
792
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
            if response_format.type == "json_object":
                self.structured_outputs.json_object = True
            elif response_format.type == "json_schema":
                json_schema = response_format.json_schema
                assert json_schema is not None
                self.structured_outputs.json = json_schema.json_schema
            elif response_format.type == "structural_tag":
                structural_tag = response_format
                assert structural_tag is not None and isinstance(
                    structural_tag,
                    (
                        LegacyStructuralTagResponseFormat,
                        StructuralTagResponseFormat,
                    ),
                )
                s_tag_obj = structural_tag.model_dump(by_alias=True)
                self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
810

811
812
813
814
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
815
        return SamplingParams.from_optional(
816
817
818
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
819
820
821
822
823
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
824
            seed=self.seed,
825
826
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
827
            logprobs=self.top_logprobs if self.logprobs else None,
828
            prompt_logprobs=prompt_logprobs,
829
            ignore_eos=self.ignore_eos,
830
            max_tokens=max_tokens,
831
            min_tokens=self.min_tokens,
832
833
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
834
835
836
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
837
            include_stop_str_in_output=self.include_stop_str_in_output,
838
            truncate_prompt_tokens=self.truncate_prompt_tokens,
839
840
841
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
842
            structured_outputs=self.structured_outputs,
Robert Shaw's avatar
Robert Shaw committed
843
            logit_bias=self.logit_bias,
844
            bad_words=self.bad_words,
845
            allowed_token_ids=self.allowed_token_ids,
846
847
            extra_args=extra_args or None,
        )
848

849
    @model_validator(mode="before")
850
    @classmethod
851
852
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
853
            raise ValueError("Stream options can only be defined when `stream=True`.")
854
855
856
857
858
859
860

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
861
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
862
                raise ValueError(
863
864
                    "`prompt_logprobs` are not available when `stream=True`."
                )
865

866
            if prompt_logprobs < 0 and prompt_logprobs != -1:
867
                raise ValueError("`prompt_logprobs` must be a positive value or -1.")
868
        if (top_logprobs := data.get("top_logprobs")) is not None:
869
            if top_logprobs < 0 and top_logprobs != -1:
870
                raise ValueError("`top_logprobs` must be a positive value or -1.")
871

872
            if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"):
873
874
875
876
877
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )

        return data
878

879
880
    @model_validator(mode="before")
    @classmethod
881
    def check_structured_outputs_count(cls, data):
882
883
884
        if isinstance(data, ValueError):
            raise data

885
        if data.get("structured_outputs", None) is None:
886
887
            return data

888
        structured_outputs_kwargs = data["structured_outputs"]
889
890
        count = sum(
            structured_outputs_kwargs.get(k) is not None
891
892
            for k in ("json", "regex", "choice")
        )
893
894
        # you can only use one kind of constraints for structured outputs
        if count > 1:
895
            raise ValueError(
896
                "You can only use one kind of constraints for structured "
897
898
                "outputs ('json', 'regex' or 'choice')."
            )
899
900
        # you can only either use structured outputs or tools, not both
        if count > 1 and data.get("tool_choice", "none") not in (
901
902
903
            "none",
            "auto",
            "required",
904
        ):
905
            raise ValueError(
906
                "You can only either use constraints for structured outputs "
907
908
                "or tools, not both."
            )
909
910
911
912
        return data

    @model_validator(mode="before")
    @classmethod
913
914
915
    def check_tool_usage(cls, data):
        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
916
        if "tool_choice" not in data and data.get("tools"):
917
918
            data["tool_choice"] = "auto"

919
        # if "tool_choice" is "none" -- no validation is needed for tools
920
921
922
        if "tool_choice" in data and data["tool_choice"] == "none":
            return data

923
        # if "tool_choice" is specified -- validation
924
        if "tool_choice" in data and data["tool_choice"] is not None:
925
            # ensure that if "tool choice" is specified, tools are present
926
            if "tools" not in data or data["tools"] is None:
927
                raise ValueError("When using `tool_choice`, `tools` must be set.")
928
929

            # make sure that tool choice is either a named tool
930
            # OR that it's set to "auto" or "required"
931
932
933
            if data["tool_choice"] not in ["auto", "required"] and not isinstance(
                data["tool_choice"], dict
            ):
934
                raise ValueError(
935
936
937
                    f"Invalid value for `tool_choice`: {data['tool_choice']}! "
                    'Only named tools, "none", "auto" or "required" '
                    "are supported."
938
                )
939

940
941
942
            # if tool_choice is "required" but the "tools" list is empty,
            # override the data to behave like "none" to align with
            # OpenAI’s behavior.
943
944
945
946
947
            if (
                data["tool_choice"] == "required"
                and isinstance(data["tools"], list)
                and len(data["tools"]) == 0
            ):
948
949
950
951
                data["tool_choice"] = "none"
                del data["tools"]
                return data

952
953
            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
954
955
            correct_usage_message = (
                'Correct usage: `{"type": "function",'
956
                ' "function": {"name": "my_function"}}`'
957
            )
958
959
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
960
961
                function = data["tool_choice"].get("function")
                if not isinstance(function, dict):
962
                    raise ValueError(
963
                        f"Invalid value for `function`: `{function}` in "
964
965
                        f"`tool_choice`! {correct_usage_message}"
                    )
966
                if "name" not in function:
967
968
969
970
                    raise ValueError(
                        f"Expected field `name` in `function` in "
                        f"`tool_choice`! {correct_usage_message}"
                    )
971
                function_name = function["name"]
972
                if not isinstance(function_name, str) or len(function_name) == 0:
973
                    raise ValueError(
974
                        f"Invalid `name` in `function`: `{function_name}`"
975
976
                        f" in `tool_choice`! {correct_usage_message}"
                    )
977
                for tool in data["tools"]:
978
                    if tool["function"]["name"] == function_name:
979
980
981
982
983
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
984
985
                        " of the specified `tools`"
                    )
986
987
        return data

988
989
990
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
991
992
993
994
995
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
996
997
        return data

998
999
1000
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
1001
1002
1003
1004
1005
1006
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
1007
1008
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1009

1010
class CompletionRequest(OpenAIBaseModel):
1011
1012
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
1013
1014
1015
1016
1017
1018
1019
    model: str | None = None
    prompt: list[int] | list[list[int]] | str | list[str] | None = None
    echo: bool | None = False
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: int | None = None
    max_tokens: int | None = 16
1020
    n: int = 1
1021
1022
1023
1024
1025
1026
1027
1028
1029
    presence_penalty: float | None = 0.0
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    suffix: str | None = None
    temperature: float | None = None
    top_p: float | None = None
    user: str | None = None
1030

1031
    # --8<-- [start:completion-sampling-params]
1032
    use_beam_search: bool = False
1033
1034
1035
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
1036
    length_penalty: float = 1.0
1037
    stop_token_ids: list[int] | None = []
1038
1039
1040
1041
1042
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
1043
1044
1045
    truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
    allowed_token_ids: list[int] | None = None
    prompt_logprobs: int | None = None
1046
    # --8<-- [end:completion-sampling-params]
1047

1048
    # --8<-- [start:completion-extra-params]
1049
    prompt_embeds: bytes | list[bytes] | None = None
1050
1051
    add_special_tokens: bool = Field(
        default=True,
1052
        description=(
1053
            "If true (the default), special tokens (e.g. BOS) will be added to "
1054
1055
            "the prompt."
        ),
1056
    )
1057
    response_format: AnyResponseFormat | None = Field(
1058
        default=None,
1059
1060
1061
1062
1063
        description=(
            "Similar to chat completion, this parameter specifies the format "
            "of output. Only {'type': 'json_object'}, {'type': 'json_schema'}"
            ", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
        ),
1064
    )
1065
    structured_outputs: StructuredOutputsParams | None = Field(
1066
        default=None,
1067
        description="Additional kwargs for structured outputs",
1068
    )
1069
1070
1071
1072
1073
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1074
1075
            "if the served model does not use priority scheduling."
        ),
1076
    )
1077
    request_id: str = Field(
1078
        default_factory=random_uuid,
1079
1080
1081
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1082
1083
            "through out the inference process and return in response."
        ),
1084
    )
1085
    logits_processors: LogitsProcessors | None = Field(
1086
1087
1088
1089
1090
1091
1092
1093
1094
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
1095
1096
1097
            "{'param': 'value'}}."
        ),
    )
1098

1099
    return_tokens_as_token_ids: bool | None = Field(
1100
1101
1102
1103
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
1104
1105
1106
            "that are not JSON-encodable can be identified."
        ),
    )
1107
    return_token_ids: bool | None = Field(
1108
1109
1110
1111
1112
1113
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
1114
1115
1116
            "need to map generated text back to input tokens."
        ),
    )
1117

1118
    cache_salt: str | None = Field(
1119
1120
1121
1122
1123
1124
1125
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
1126
            "to 256 bit)."
1127
1128
        ),
    )
1129

1130
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
1131
        default=None,
1132
1133
        description="KVTransfer parameters used for disaggregated serving.",
    )
Robert Shaw's avatar
Robert Shaw committed
1134

1135
    vllm_xargs: dict[str, str | int | float] | None = Field(
1136
        default=None,
1137
1138
1139
1140
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
1141
1142
    )

1143
    # --8<-- [end:completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
1144

1145
1146
1147
1148
1149
    # Default sampling parameters for completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
1150
        "top_k": 0,
1151
1152
1153
1154
        "min_p": 0.0,
    }

    def to_beam_search_params(
1155
1156
        self,
        max_tokens: int,
1157
        default_sampling_params: dict | None = None,
1158
1159
1160
    ) -> BeamSearchParams:
        if default_sampling_params is None:
            default_sampling_params = {}
1161
        n = self.n if self.n is not None else 1
1162
1163
1164

        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get("temperature", 1.0)
1165
1166
1167
1168
1169
1170

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
1171
            length_penalty=self.length_penalty,
1172
1173
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
1174

1175
    def to_sampling_params(
1176
        self,
1177
        max_tokens: int,
1178
1179
        logits_processor_pattern: str | None,
        default_sampling_params: dict | None = None,
1180
    ) -> SamplingParams:
1181
1182
        if default_sampling_params is None:
            default_sampling_params = {}
1183

1184
1185
1186
1187
1188
1189
1190
1191
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
1192
1193
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
1194
1195
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
1196
1197
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
1198
1199
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
1200
1201
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
1202
1203
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
1204
1205
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
1206

1207
1208
1209
1210
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

1211
1212
        echo_without_generation = self.echo and self.max_tokens == 0

1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
        response_format = self.response_format
        if response_format is not None:
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
            if response_format.type == "json_object":
                self.structured_outputs.json_object = True
            elif response_format.type == "json_schema":
                json_schema = response_format.json_schema
1225
                assert json_schema is not None
1226
1227
1228
                self.structured_outputs.json = json_schema.json_schema
            elif response_format.type == "structural_tag":
                structural_tag = response_format
1229
                assert structural_tag is not None and isinstance(
1230
1231
1232
1233
1234
                    structural_tag,
                    (
                        LegacyStructuralTagResponseFormat,
                        StructuralTagResponseFormat,
                    ),
1235
1236
                )
                s_tag_obj = structural_tag.model_dump(by_alias=True)
1237
                self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
1238

1239
1240
1241
1242
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
1243
        return SamplingParams.from_optional(
1244
1245
1246
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
1247
1248
1249
1250
1251
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
1252
            seed=self.seed,
1253
1254
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
1255
            logprobs=self.logprobs,
1256
            ignore_eos=self.ignore_eos,
1257
            max_tokens=max_tokens if not echo_without_generation else 1,
1258
            min_tokens=self.min_tokens,
1259
            prompt_logprobs=prompt_logprobs,
1260
            skip_special_tokens=self.skip_special_tokens,
1261
            spaces_between_special_tokens=self.spaces_between_special_tokens,
1262
            include_stop_str_in_output=self.include_stop_str_in_output,
1263
1264
1265
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
1266
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1267
1268
1269
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
1270
            structured_outputs=self.structured_outputs,
1271
            logit_bias=self.logit_bias,
Robert Shaw's avatar
Robert Shaw committed
1272
            allowed_token_ids=self.allowed_token_ids,
1273
            extra_args=extra_args or None,
1274
        )
1275

1276
1277
    @model_validator(mode="before")
    @classmethod
1278
    def check_structured_outputs_count(cls, data):
1279
        if data.get("structured_outputs", None) is None:
1280
1281
            return data

1282
        structured_outputs_kwargs = data["structured_outputs"]
1283
1284
        count = sum(
            structured_outputs_kwargs.get(k) is not None
1285
1286
            for k in ("json", "regex", "choice")
        )
1287
        if count > 1:
1288
            raise ValueError(
1289
                "You can only use one kind of constraints for structured "
1290
1291
                "outputs ('json', 'regex' or 'choice')."
            )
1292
1293
        return data

1294
1295
1296
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
1297
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
1298
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
1299
                raise ValueError(
1300
1301
                    "`prompt_logprobs` are not available when `stream=True`."
                )
1302

1303
            if prompt_logprobs < 0 and prompt_logprobs != -1:
1304
                raise ValueError("`prompt_logprobs` must be a positive value or -1.")
1305
1306
1307
        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
            raise ValueError("`logprobs` must be a positive value.")

1308
1309
        return data

1310
1311
1312
1313
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
1314
            raise ValueError("Stream options can only be defined when `stream=True`.")
1315

1316
1317
        return data

1318
1319
1320
    @model_validator(mode="before")
    @classmethod
    def validate_prompt_and_prompt_embeds(cls, data):
1321
1322
1323
        prompt = data.get("prompt")
        prompt_embeds = data.get("prompt_embeds")

1324
1325
1326
1327
        prompt_is_empty = prompt is None or (isinstance(prompt, str) and prompt == "")
        embeds_is_empty = prompt_embeds is None or (
            isinstance(prompt_embeds, list) and len(prompt_embeds) == 0
        )
1328
1329

        if prompt_is_empty and embeds_is_empty:
1330
            raise ValueError(
1331
1332
1333
                "Either prompt or prompt_embeds must be provided and non-empty."
            )

1334
1335
        return data

1336
1337
1338
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
1339
1340
1341
1342
1343
1344
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
1345
1346
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1347

1348
class CompletionLogProbs(OpenAIBaseModel):
1349
    text_offset: list[int] = Field(default_factory=list)
1350
    token_logprobs: list[float | None] = Field(default_factory=list)
1351
    tokens: list[str] = Field(default_factory=list)
1352
    top_logprobs: list[dict[str, float] | None] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
1353
1354


1355
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1356
1357
    index: int
    text: str
1358
1359
1360
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1361
1362
1363
1364
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1365
1366
            "including encountering the EOS token"
        ),
1367
    )
1368
1369
1370
    token_ids: list[int] | None = None  # For response
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None  # For prompt
Zhuohan Li's avatar
Zhuohan Li committed
1371
1372


1373
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1374
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
1375
    object: Literal["text_completion"] = "text_completion"
Zhuohan Li's avatar
Zhuohan Li committed
1376
1377
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1378
    choices: list[CompletionResponseChoice]
1379
1380
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1381
    usage: UsageInfo
1382
1383

    # vLLM-specific fields that are not in OpenAI spec
1384
    kv_transfer_params: dict[str, Any] | None = Field(
1385
1386
        default=None, description="KVTransfer parameters."
    )
Zhuohan Li's avatar
Zhuohan Li committed
1387
1388


1389
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1390
1391
    index: int
    text: str
1392
1393
1394
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1395
1396
1397
1398
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1399
1400
            "including encountering the EOS token"
        ),
1401
    )
1402
1403
    # not part of the OpenAI spec but for tracing the tokens
    # prompt tokens is put into choice to align with CompletionResponseChoice
1404
1405
    prompt_token_ids: list[int] | None = None
    token_ids: list[int] | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1406
1407


1408
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1409
1410
1411
1412
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1413
    choices: list[CompletionResponseStreamChoice]
1414
    usage: UsageInfo | None = Field(default=None)
1415
1416


1417
1418
1419
1420
1421
1422
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
1423
    id: str = Field(default_factory=make_tool_call_id)
1424
1425
1426
1427
    type: Literal["function"] = "function"
    function: FunctionCall


1428
class DeltaFunctionCall(BaseModel):
1429
1430
    name: str | None = None
    arguments: str | None = None
1431
1432
1433
1434


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
1435
1436
    id: str | None = None
    type: Literal["function"] | None = None
1437
    index: int
1438
    function: DeltaFunctionCall | None = None
1439
1440
1441
1442
1443
1444
1445


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
1446
    tool_calls: list[ToolCall]
1447
1448
1449

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
1450
    content: str | None = None
1451
1452


1453
class ChatMessage(OpenAIBaseModel):
1454
    role: str
1455
1456
1457
1458
1459
    content: str | None = None
    refusal: str | None = None
    annotations: OpenAIAnnotation | None = None
    audio: OpenAIChatCompletionAudio | None = None
    function_call: FunctionCall | None = None
1460
    tool_calls: list[ToolCall] = Field(default_factory=list)
1461

1462
    # vLLM-specific fields that are not in OpenAI spec
1463
    reasoning: str | None = None
1464
    reasoning_content: str | None = None
1465
1466
1467
1468
1469
1470
1471
    """Deprecated: use `reasoning` instead."""

    @model_validator(mode="after")
    def handle_deprecated_reasoning_content(self):
        """Copy reasoning to reasoning_content for backward compatibility."""
        self.reasoning_content = self.reasoning
        return self
1472

1473

1474
1475
1476
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
1477
    bytes: list[int] | None = None
1478
1479
1480


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
1481
1482
    # Workaround: redefine fields name cache so that it's not
    # shared with the super class.
1483
    field_names: ClassVar[set[str] | None] = None
1484
    top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
1485
1486
1487


class ChatCompletionLogProbs(OpenAIBaseModel):
1488
    content: list[ChatCompletionLogProbsContent] | None = None
1489
1490


1491
class ChatCompletionResponseChoice(OpenAIBaseModel):
1492
1493
    index: int
    message: ChatMessage
1494
    logprobs: ChatCompletionLogProbs | None = None
1495
    # per OpenAI spec this is the default
1496
    finish_reason: str | None = "stop"
1497
    # not part of the OpenAI spec but included in vLLM for legacy reasons
1498
    stop_reason: int | str | None = None
1499
1500
    # not part of the OpenAI spec but is useful for tracing the tokens
    # in agent scenarios
1501
    token_ids: list[int] | None = None
1502
1503


1504
class ChatCompletionResponse(OpenAIBaseModel):
1505
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
1506
    object: Literal["chat.completion"] = "chat.completion"
1507
1508
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1509
    choices: list[ChatCompletionResponseChoice]
1510
1511
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
1512
    usage: UsageInfo
1513
1514

    # vLLM-specific fields that are not in OpenAI spec
1515
1516
1517
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None
    kv_transfer_params: dict[str, Any] | None = Field(
1518
1519
        default=None, description="KVTransfer parameters."
    )
1520
1521


1522
class DeltaMessage(OpenAIBaseModel):
1523
1524
    role: str | None = None
    content: str | None = None
1525
    reasoning: str | None = None
1526
    reasoning_content: str | None = None
1527
    """Deprecated: use `reasoning` instead."""
1528
    tool_calls: list[DeltaToolCall] = Field(default_factory=list)
1529

1530
1531
1532
1533
1534
1535
    @model_validator(mode="after")
    def handle_deprecated_reasoning_content(self):
        """Copy reasoning to reasoning_content for backward compatibility."""
        self.reasoning_content = self.reasoning
        return self

1536

1537
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
1538
1539
    index: int
    delta: DeltaMessage
1540
1541
1542
    logprobs: ChatCompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = None
1543
    # not part of the OpenAI spec but for tracing the tokens
1544
    token_ids: list[int] | None = None
1545
1546


1547
class ChatCompletionStreamResponse(OpenAIBaseModel):
1548
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
1549
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
1550
1551
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1552
    choices: list[ChatCompletionResponseStreamChoice]
1553
    usage: UsageInfo | None = Field(default=None)
1554
    # not part of the OpenAI spec but for tracing the tokens
1555
    prompt_token_ids: list[int] | None = None
1556
1557


1558
1559
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
1560
1561
    finish_reason: str | None = None
    stop_reason: int | str | None = None
1562
1563
1564
1565
1566
1567
1568
1569


class TranscriptionStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
    object: Literal["transcription.chunk"] = "transcription.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranscriptionResponseStreamChoice]
1570
    usage: UsageInfo | None = Field(default=None)
1571
1572


1573
1574
class InputTokensDetails(OpenAIBaseModel):
    cached_tokens: int
1575
1576
    input_tokens_per_turn: list[int] = Field(default_factory=list)
    cached_tokens_per_turn: list[int] = Field(default_factory=list)
1577
1578
1579


class OutputTokensDetails(OpenAIBaseModel):
1580
1581
    reasoning_tokens: int = 0
    tool_output_tokens: int = 0
1582
1583
    output_tokens_per_turn: list[int] = Field(default_factory=list)
    tool_output_tokens_per_turn: list[int] = Field(default_factory=list)
1584
1585
1586
1587
1588
1589
1590
1591


class ResponseUsage(OpenAIBaseModel):
    input_tokens: int
    input_tokens_details: InputTokensDetails
    output_tokens: int
    output_tokens_details: OutputTokensDetails
    total_tokens: int
1592
1593


1594
1595
1596
1597
1598
1599
def serialize_message(msg):
    """
    Serializes a single message
    """
    if isinstance(msg, dict):
        return msg
1600
    elif hasattr(msg, "to_dict"):
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
        return msg.to_dict()
    else:
        # fallback to pyandic dump
        return msg.model_dump_json()


def serialize_messages(msgs):
    """
    Serializes multiple messages
    """
    return [serialize_message(msg) for msg in msgs] if msgs else None


1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
class ResponseRawMessageAndToken(OpenAIBaseModel):
    """Class to show the raw message.
    If message / tokens diverge, tokens is the source of truth"""

    message: str
    tokens: list[int]
    type: Literal["raw_message_tokens"] = "raw_message_tokens"


ResponseInputOutputMessage: TypeAlias = (
    list[ChatCompletionMessageParam] | list[ResponseRawMessageAndToken]
)


1628
1629
1630
1631
class ResponsesResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
    created_at: int = Field(default_factory=lambda: int(time.time()))
    # error: Optional[ResponseError] = None
1632
1633
1634
    incomplete_details: IncompleteDetails | None = None
    instructions: str | None = None
    metadata: Metadata | None = None
1635
1636
    model: str
    object: Literal["response"] = "response"
1637
    output: list[ResponseOutputItem]
1638
1639
1640
1641
1642
1643
1644
    parallel_tool_calls: bool
    temperature: float
    tool_choice: ToolChoice
    tools: list[Tool]
    top_p: float
    background: bool
    max_output_tokens: int
1645
1646
1647
1648
    max_tool_calls: int | None = None
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
1649
1650
    service_tier: Literal["auto", "default", "flex", "scale", "priority"]
    status: ResponseStatus
1651
1652
    text: ResponseTextConfig | None = None
    top_logprobs: int | None = None
1653
    truncation: Literal["auto", "disabled"]
1654
1655
    usage: ResponseUsage | None = None
    user: str | None = None
1656

1657
1658
1659
1660
    # --8<-- [start:responses-extra-params]
    # These are populated when enable_response_messages is set to True
    # NOTE: custom serialization is needed
    # see serialize_input_messages and serialize_output_messages
1661
1662
    input_messages: ResponseInputOutputMessage | None = None
    output_messages: ResponseInputOutputMessage | None = None
1663
1664
1665
1666
1667
1668
1669
    # --8<-- [end:responses-extra-params]

    # NOTE: openAI harmony doesn't serialize TextContent properly,
    # TODO: this fixes for TextContent, but need to verify for tools etc
    # https://github.com/openai/harmony/issues/78
    @field_serializer("output_messages", when_used="json")
    def serialize_output_messages(self, msgs, _info):
1670
        return serialize_messages(msgs)
1671
1672
1673
1674
1675

    # NOTE: openAI harmony doesn't serialize TextContent properly, this fixes it
    # https://github.com/openai/harmony/issues/78
    @field_serializer("input_messages", when_used="json")
    def serialize_input_messages(self, msgs, _info):
1676
        return serialize_messages(msgs)
1677

1678
1679
1680
1681
1682
1683
1684
1685
1686
    @classmethod
    def from_request(
        cls,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
        model_name: str,
        created_time: int,
        output: list[ResponseOutputItem],
        status: ResponseStatus,
1687
        usage: ResponseUsage | None = None,
1688
1689
        input_messages: ResponseInputOutputMessage | None = None,
        output_messages: ResponseInputOutputMessage | None = None,
1690
    ) -> "ResponsesResponse":
1691
        incomplete_details: IncompleteDetails | None = None
1692
1693
        if status == "incomplete":
            incomplete_details = IncompleteDetails(reason="max_output_tokens")
1694
1695
1696
        # TODO: implement the other reason for incomplete_details,
        # which is content_filter
        # incomplete_details = IncompleteDetails(reason='content_filter')
1697
1698
1699
        return cls(
            id=request.request_id,
            created_at=created_time,
1700
            incomplete_details=incomplete_details,
1701
1702
1703
1704
            instructions=request.instructions,
            metadata=request.metadata,
            model=model_name,
            output=output,
1705
1706
            input_messages=input_messages,
            output_messages=output_messages,
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
            parallel_tool_calls=request.parallel_tool_calls,
            temperature=sampling_params.temperature,
            tool_choice=request.tool_choice,
            tools=request.tools,
            top_p=sampling_params.top_p,
            background=request.background,
            max_output_tokens=sampling_params.max_tokens,
            max_tool_calls=request.max_tool_calls,
            previous_response_id=request.previous_response_id,
            prompt=request.prompt,
            reasoning=request.reasoning,
            service_tier=request.service_tier,
            status=status,
            text=request.text,
            top_logprobs=sampling_params.logprobs,
            truncation=request.truncation,
            user=request.user,
            usage=usage,
        )


1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartDoneEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.done"]
    """The type of the event. Always `response.reasoning_part.done`."""


# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartAddedEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.added"]
    """The type of the event. Always `response.reasoning_part.added`."""


1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
# vLLM Streaming Events
# Note: we override the response type with the vLLM ResponsesResponse type
class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseInProgressEvent(OpenAIResponseInProgressEvent):
    response: ResponsesResponse  # type: ignore[override]


1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
StreamingResponsesResponse: TypeAlias = (
    ResponseCreatedEvent
    | ResponseInProgressEvent
    | ResponseCompletedEvent
    | ResponseOutputItemAddedEvent
    | ResponseOutputItemDoneEvent
    | ResponseContentPartAddedEvent
    | ResponseContentPartDoneEvent
    | ResponseReasoningTextDeltaEvent
    | ResponseReasoningTextDoneEvent
    | ResponseReasoningPartAddedEvent
    | ResponseReasoningPartDoneEvent
    | ResponseCodeInterpreterCallInProgressEvent
    | ResponseCodeInterpreterCallCodeDeltaEvent
    | ResponseWebSearchCallInProgressEvent
    | ResponseWebSearchCallSearchingEvent
    | ResponseWebSearchCallCompletedEvent
    | ResponseCodeInterpreterCallCodeDoneEvent
    | ResponseCodeInterpreterCallInterpretingEvent
    | ResponseCodeInterpreterCallCompletedEvent
1806
1807
1808
1809
    | ResponseMcpCallArgumentsDeltaEvent
    | ResponseMcpCallArgumentsDoneEvent
    | ResponseMcpCallInProgressEvent
    | ResponseMcpCallCompletedEvent
1810
)
1811

1812

1813
class TokenizeCompletionRequest(OpenAIBaseModel):
1814
    model: str | None = None
1815
1816
    prompt: str

1817
1818
1819
1820
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
1821
1822
            "the prompt."
        ),
1823
    )
1824
    return_token_strs: bool | None = Field(
1825
        default=False,
1826
1827
1828
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
1829
    )
1830
1831
1832


class TokenizeChatRequest(OpenAIBaseModel):
1833
    model: str | None = None
1834
    messages: list[ChatCompletionMessageParam]
1835

1836
1837
    add_generation_prompt: bool = Field(
        default=True,
1838
1839
1840
1841
1842
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
1843
    )
1844
    return_token_strs: bool | None = Field(
1845
        default=False,
1846
1847
1848
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
1849
    )
1850
1851
    continue_final_message: bool = Field(
        default=False,
1852
1853
1854
1855
1856
1857
1858
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
1859
1860
1861
1862
1863
1864
1865
1866
    )
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
1867
1868
            "default)."
        ),
1869
    )
1870
    chat_template: str | None = Field(
1871
1872
1873
1874
1875
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
1876
1877
            "does not define one."
        ),
1878
    )
1879
    chat_template_kwargs: dict[str, Any] | None = Field(
1880
        default=None,
1881
1882
        description=(
            "Additional keyword args to pass to the template renderer. "
1883
1884
            "Will be accessible by the chat template."
        ),
1885
    )
1886
    mm_processor_kwargs: dict[str, Any] | None = Field(
1887
1888
1889
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
1890
    tools: list[ChatCompletionToolsParam] | None = Field(
1891
1892
1893
        default=None,
        description=("A list of tools the model may call."),
    )
1894

1895
1896
1897
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
1898
1899
1900
1901
1902
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
1903
1904
        return data

1905

1906
TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest
1907
1908
1909
1910
1911


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
1912
    tokens: list[int]
1913
    token_strs: list[str] | None = None
1914
1915
1916


class DetokenizeRequest(OpenAIBaseModel):
1917
    model: str | None = None
1918
    tokens: list[int]
1919
1920
1921
1922


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
1923
1924


1925
1926
class TokenizerInfoResponse(OpenAIBaseModel):
    """
1927
    Response containing tokenizer configuration
1928
1929
1930
1931
1932
1933
1934
    equivalent to tokenizer_config.json
    """

    model_config = ConfigDict(extra="allow")
    tokenizer_class: str


1935
class LoadLoRAAdapterRequest(BaseModel):
1936
1937
1938
1939
    lora_name: str
    lora_path: str


1940
class UnloadLoRAAdapterRequest(BaseModel):
1941
    lora_name: str
1942
    lora_int_id: int | None = Field(default=None)
1943
1944
1945


## Protocols for Audio
1946
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
1947
1948
1949
1950


class TranscriptionRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
1951
    # https://platform.openai.com/docs/api-reference/audio/createTranscription
1952
1953
1954
1955
1956
1957
1958

    file: UploadFile
    """
    The audio file object (not file name) to transcribe, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

1959
    model: str | None = None
1960
1961
1962
    """ID of the model to use.
    """

1963
    language: str | None = None
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
    """The language of the input audio.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy and latency.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    ## TODO (varun) : Support if set to 0, certain thresholds are met !!

1987
    timestamp_granularities: list[Literal["word", "segment"]] = Field(
1988
1989
        alias="timestamp_granularities[]", default=[]
    )
1990
1991
1992
1993
1994
1995
1996
1997
    """The timestamp granularities to populate for this transcription.

    `response_format` must be set `verbose_json` to use timestamp granularities.
    Either or both of these options are supported: `word`, or `segment`. Note:
    There is no additional latency for segment timestamps, but generating word
    timestamps incurs additional latency.
    """

1998
    stream: bool | None = False
1999
    """When set, it will enable output to be streamed in a similar fashion
2000
    as the Chat Completion endpoint.
2001
    """
2002
    # --8<-- [start:transcription-extra-params]
2003
    # Flattened stream option to simplify form data.
2004
2005
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2006

2007
    vllm_xargs: dict[str, str | int | float] | None = Field(
2008
        default=None,
2009
2010
2011
2012
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
2013
    )
2014
    # --8<-- [end:transcription-extra-params]
2015

2016
    to_language: str | None = None
2017
2018
    """The language of the output audio we transcribe to.

2019
    Please note that this is not currently used by supported models at this
2020
2021
2022
    time, but it is a placeholder for future use, matching translation api.
    """

2023
    # --8<-- [start:transcription-sampling-params]
2024
2025
2026
2027
2028
2029
2030
2031
2032
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """

2033
    top_p: float | None = None
2034
    """Enables nucleus (top-p) sampling, where tokens are selected from the
2035
2036
2037
    smallest possible set whose cumulative probability exceeds `p`.
    """

2038
    top_k: int | None = None
2039
2040
    """Limits sampling to the `k` most probable tokens at each step."""

2041
    min_p: float | None = None
2042
    """Filters out tokens with a probability lower than `min_p`, ensuring a
2043
2044
2045
    minimum likelihood threshold during sampling.
    """

2046
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2047
2048
    """The seed to use for sampling."""

2049
    frequency_penalty: float | None = 0.0
2050
2051
    """The frequency penalty to use for sampling."""

2052
    repetition_penalty: float | None = None
2053
2054
    """The repetition penalty to use for sampling."""

2055
    presence_penalty: float | None = 0.0
2056
    """The presence penalty to use for sampling."""
2057
2058
2059

    max_completion_tokens: int | None = None
    """The maximum number of tokens to generate."""
2060
    # --8<-- [end:transcription-sampling-params]
2061

2062
2063
    # Default sampling parameters for transcription requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
2064
2065
2066
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
2067
        "top_k": 0,
2068
        "min_p": 0.0,
2069
2070
2071
    }

    def to_sampling_params(
2072
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2073
    ) -> SamplingParams:
2074
2075
2076
2077
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
2078

2079
2080
2081
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2082
2083
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2084
2085
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
2086
2087
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
2088
2089
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
2090
2091
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
2092
2093
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
2094
2095
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
2096
2097
2098
2099

        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )

        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=repetition_penalty,
            presence_penalty=self.presence_penalty,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
            extra_args=self.vllm_xargs,
        )
2118
2119
2120

    @model_validator(mode="before")
    @classmethod
2121
2122
2123
2124
2125
2126
2127
    def validate_transcription_request(cls, data):
        if isinstance(data.get("file"), str):
            raise HTTPException(
                status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
                detail="Expected 'file' to be a file-like object, not 'str'.",
            )

2128
2129
2130
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
2131
            raise ValueError("Stream options can only be defined when `stream=True`.")
2132
2133

        return data
2134
2135
2136


# Transcription response objects
2137
2138
2139
2140
2141
class TranscriptionUsageAudio(OpenAIBaseModel):
    type: Literal["duration"] = "duration"
    seconds: int


2142
2143
2144
class TranscriptionResponse(OpenAIBaseModel):
    text: str
    """The transcribed text."""
2145
    usage: TranscriptionUsageAudio
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162


class TranscriptionWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranscriptionSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

2163
    avg_logprob: float | None = None
2164
2165
2166
2167
2168
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

2169
    compression_ratio: float | None = None
2170
2171
2172
2173
2174
2175
2176
2177
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

2178
    no_speech_prob: float | None = None
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

2197
    tokens: list[int]
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
    """Array of token IDs for the text content."""


class TranscriptionResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The transcribed text."""

2211
    segments: list[TranscriptionSegment] | None = None
2212
2213
    """Segments of the transcribed text and their corresponding details."""

2214
    words: list[TranscriptionWord] | None = None
2215
    """Extracted words and their corresponding timestamps."""
2216
2217


2218
2219
2220
2221
2222
TranscriptionResponseVariant: TypeAlias = (
    TranscriptionResponse | TranscriptionResponseVerbose
)


2223
2224
class TranslationResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
2225
2226
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2227
2228
2229
2230
2231
2232
2233
2234


class TranslationStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
    object: Literal["translation.chunk"] = "translation.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranslationResponseStreamChoice]
2235
    usage: UsageInfo | None = Field(default=None)
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247


class TranslationRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/audio/createTranslation

    file: UploadFile
    """
    The audio file object (not file name) to translate, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

2248
    model: str | None = None
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
    """ID of the model to use.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    # TODO support additional sampling parameters
    # --8<-- [start:translation-sampling-params]
2268
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2269
2270
    """The seed to use for sampling."""

2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """
    # --8<-- [end:translation-sampling-params]

    # --8<-- [start:translation-extra-params]
2282
    language: str | None = None
2283
2284
2285
2286
2287
2288
2289
    """The language of the input audio we translate from.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy.
    """

2290
    to_language: str | None = None
2291
2292
2293
2294
2295
2296
2297
    """The language of the input audio we translate to.

    Please note that this is not supported by all models, refer to the specific
    model documentation for more details.
    For instance, Whisper only supports `to_language=en`.
    """

2298
    stream: bool | None = False
2299
    """Custom field not present in the original OpenAI definition. When set,
2300
    it will enable output to be streamed in a similar fashion as the Chat
2301
    Completion endpoint.
2302
2303
    """
    # Flattened stream option to simplify form data.
2304
2305
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2306
2307
2308

    max_completion_tokens: int | None = None
    """The maximum number of tokens to generate."""
2309
2310
2311
2312
2313
2314
2315
2316
    # --8<-- [end:translation-extra-params]

    # Default sampling parameters for translation requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "temperature": 0,
    }

    def to_sampling_params(
2317
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2318
    ) -> SamplingParams:
2319
2320
2321
2322
2323
2324
2325
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2326
2327
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2328

2329
2330
2331
2332
2333
2334
2335
2336
        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
        )
2337
2338
2339
2340
2341
2342
2343

    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
2344
            raise ValueError("Stream options can only be defined when `stream=True`.")
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369

        return data


# Translation response objects
class TranslationResponse(OpenAIBaseModel):
    text: str
    """The translated text."""


class TranslationWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranslationSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

2370
    avg_logprob: float | None = None
2371
2372
2373
2374
2375
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

2376
    compression_ratio: float | None = None
2377
2378
2379
2380
2381
2382
2383
2384
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

2385
    no_speech_prob: float | None = None
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

    tokens: list[int]
    """Array of token IDs for the text content."""


class TranslationResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The translated text."""

2418
    segments: list[TranslationSegment] | None = None
2419
2420
    """Segments of the translated text and their corresponding details."""

2421
    words: list[TranslationWord] | None = None
2422
    """Extracted words and their corresponding timestamps."""
2423
2424


2425
2426
2427
TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose


2428
2429
2430
####### Tokens IN <> Tokens OUT #######
class GenerateRequest(BaseModel):
    request_id: str = Field(
2431
        default_factory=random_uuid,
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    token_ids: list[int]
    """The token ids to generate text from."""

    # features: MultiModalFeatureSpec
    # TODO (NickLucche): implement once Renderer work is completed
    features: str | None = None
    """The processed MM inputs for the model."""

    sampling_params: SamplingParams
    """The sampling parameters for the model."""

    model: str | None = None

    stream: bool | None = False
    stream_options: StreamOptions | None = None
    cache_salt: str | None = Field(
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
            "to 256 bit)."
        ),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."
        ),
    )
    kv_transfer_params: dict[str, Any] | None = Field(
        default=None,
        description="KVTransfer parameters used for disaggregated serving.",
    )


class GenerateResponseChoice(BaseModel):
    index: int
    logprobs: ChatCompletionLogProbs | None = None
    # per OpenAI spec this is the default
    finish_reason: str | None = "stop"
    token_ids: list[int] | None = None


class GenerateResponse(BaseModel):
    request_id: str = Field(
2488
        default_factory=random_uuid,
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    choices: list[GenerateResponseChoice]

    prompt_logprobs: list[dict[int, Logprob] | None] | None = None

    kv_transfer_params: dict[str, Any] | None = Field(
        default=None,
        description="KVTransfer parameters used for disaggregated serving.",
    )