protocol.py 89.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import json
Zhuohan Li's avatar
Zhuohan Li committed
7
import time
8
from http import HTTPStatus
9
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
Zhuohan Li's avatar
Zhuohan Li committed
10

11
import regex as re
12
import torch
13
from fastapi import HTTPException, UploadFile
14
from openai.types.chat.chat_completion_audio import (
15
16
17
    ChatCompletionAudio as OpenAIChatCompletionAudio,
)
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
18
19
20
21
22
from openai.types.responses import (
    ResponseCodeInterpreterCallCodeDeltaEvent,
    ResponseCodeInterpreterCallCodeDoneEvent,
    ResponseCodeInterpreterCallCompletedEvent,
    ResponseCodeInterpreterCallInProgressEvent,
23
24
25
26
27
    ResponseCodeInterpreterCallInterpretingEvent,
    ResponseContentPartAddedEvent,
    ResponseContentPartDoneEvent,
    ResponseFunctionToolCall,
    ResponseInputItemParam,
28
29
30
31
    ResponseMcpCallArgumentsDeltaEvent,
    ResponseMcpCallArgumentsDoneEvent,
    ResponseMcpCallCompletedEvent,
    ResponseMcpCallInProgressEvent,
32
33
34
35
36
37
38
39
40
41
42
    ResponseOutputItem,
    ResponseOutputItemAddedEvent,
    ResponseOutputItemDoneEvent,
    ResponsePrompt,
    ResponseReasoningTextDeltaEvent,
    ResponseReasoningTextDoneEvent,
    ResponseStatus,
    ResponseWebSearchCallCompletedEvent,
    ResponseWebSearchCallInProgressEvent,
    ResponseWebSearchCallSearchingEvent,
)
43
from openai.types.responses import (
44
45
46
    ResponseCompletedEvent as OpenAIResponseCompletedEvent,
)
from openai.types.responses import ResponseCreatedEvent as OpenAIResponseCreatedEvent
47
from openai.types.responses import (
48
49
    ResponseInProgressEvent as OpenAIResponseInProgressEvent,
)
50
from openai.types.responses.response_reasoning_item import (
51
52
    Content as ResponseReasoningTextContent,
)
53
from openai_harmony import Message as OpenAIHarmonyMessage
54
55
56
57
58

# Backward compatibility for OpenAI client versions
try:  # For older openai versions (< 1.100.0)
    from openai.types.responses import ResponseTextConfig
except ImportError:  # For newer openai versions (>= 1.100.0)
59
    from openai.types.responses import ResponseFormatTextConfig as ResponseTextConfig
60

61

62
from openai.types.responses.response import IncompleteDetails, ToolChoice
63
64
from openai.types.responses.tool import Tool
from openai.types.shared import Metadata, Reasoning
65
66
67
68
from pydantic import (
    BaseModel,
    ConfigDict,
    Field,
69
    ValidationError,
70
    field_serializer,
71
72
    model_validator,
)
Zhuohan Li's avatar
Zhuohan Li committed
73

74
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam, make_tool_call_id
75
from vllm.exceptions import VLLMValidationError
76
from vllm.logger import init_logger
77
from vllm.logprobs import Logprob
78
79
80
81
82
83
from vllm.sampling_params import (
    BeamSearchParams,
    RequestOutputKind,
    SamplingParams,
    StructuredOutputsParams,
)
84
85
from vllm.utils import random_uuid
from vllm.utils.import_utils import resolve_obj_by_qualname
86

87
88
logger = init_logger(__name__)

89
_LONG_INFO = torch.iinfo(torch.long)
90

Zhuohan Li's avatar
Zhuohan Li committed
91

92
class OpenAIBaseModel(BaseModel):
93
94
95
    # OpenAI API does allow extra fields
    model_config = ConfigDict(extra="allow")

96
    # Cache class field names
97
    field_names: ClassVar[set[str] | None] = None
98

99
    @model_validator(mode="wrap")
100
    @classmethod
101
102
103
104
    def __log_extra_fields__(cls, data, handler):
        result = handler(data)
        if not isinstance(data, dict):
            return result
105
106
        field_names = cls.field_names
        if field_names is None:
107
108
109
110
            # Get all class field names and their potential aliases
            field_names = set()
            for field_name, field in cls.model_fields.items():
                field_names.add(field_name)
111
                if alias := getattr(field, "alias", None):
112
113
114
115
116
117
                    field_names.add(alias)
            cls.field_names = field_names

        # Compare against both field names and aliases
        if any(k not in field_names for k in data):
            logger.warning(
118
                "The following fields were present in the request but ignored: %s",
119
120
                data.keys() - field_names,
            )
121
        return result
122
123


124
class ErrorInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
125
126
    message: str
    type: str
127
    param: str | None = None
128
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
129
130


131
132
133
134
class ErrorResponse(OpenAIBaseModel):
    error: ErrorInfo


135
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
136
137
138
139
140
141
142
143
144
145
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
146
    group: str | None = None
147
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
148
149


150
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
151
152
153
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
154
    owned_by: str = "vllm"
155
156
157
    root: str | None = None
    parent: str | None = None
    max_model_len: int | None = None
158
    permission: list[ModelPermission] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
159
160


161
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
162
    object: str = "list"
163
    data: list[ModelCard] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
164
165


166
class PromptTokenUsageInfo(OpenAIBaseModel):
167
    cached_tokens: int | None = None
168
169


170
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
171
172
    prompt_tokens: int = 0
    total_tokens: int = 0
173
174
    completion_tokens: int | None = 0
    prompt_tokens_details: PromptTokenUsageInfo | None = None
Zhuohan Li's avatar
Zhuohan Li committed
175
176


177
178
class RequestResponseMetadata(BaseModel):
    request_id: str
179
    final_usage_info: UsageInfo | None = None
180
181


182
183
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
184
    description: str | None = None
185
186
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
187
188
    json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
    strict: bool | None = None
189
190


191
class LegacyStructuralTag(OpenAIBaseModel):
192
193
194
    begin: str
    # schema is the field, but that causes conflicts with pydantic so
    # instead use structural_tag_schema with an alias
195
    structural_tag_schema: dict[str, Any] | None = Field(default=None, alias="schema")
196
197
198
    end: str


199
class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
200
    type: Literal["structural_tag"]
201
    structures: list[LegacyStructuralTag]
202
203
204
    triggers: list[str]


205
206
207
208
209
210
211
212
213
214
class StructuralTagResponseFormat(OpenAIBaseModel):
    type: Literal["structural_tag"]
    format: Any


AnyStructuralTagResponseFormat: TypeAlias = (
    LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
)


215
class ResponseFormat(OpenAIBaseModel):
216
    # type must be "json_schema", "json_object", or "text"
217
    type: Literal["text", "json_object", "json_schema"]
218
    json_schema: JsonSchemaResponseFormat | None = None
219
220


221
222
223
AnyResponseFormat: TypeAlias = (
    ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
)
224
225


226
class StreamOptions(OpenAIBaseModel):
227
228
    include_usage: bool | None = True
    continuous_usage_stats: bool | None = False
229
230


231
232
class FunctionDefinition(OpenAIBaseModel):
    name: str
233
234
    description: str | None = None
    parameters: dict[str, Any] | None = None
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


251
252
# extra="forbid" is a workaround to have kwargs as a field,
# see https://github.com/pydantic/pydantic/issues/3125
253
254
class LogitsProcessorConstructor(BaseModel):
    qualname: str
255
256
    args: list[Any] | None = None
    kwargs: dict[str, Any] | None = None
257

258
259
    model_config = ConfigDict(extra="forbid")

260

261
LogitsProcessors = list[str | LogitsProcessorConstructor]
262
263


264
def get_logits_processors(
265
266
    processors: LogitsProcessors | None, pattern: str | None
) -> list[Any] | None:
267
268
269
    if processors and pattern:
        logits_processors = []
        for processor in processors:
270
            qualname = processor if isinstance(processor, str) else processor.qualname
271
272
273
274
            if not re.match(pattern, qualname):
                raise ValueError(
                    f"Logits processor '{qualname}' is not allowed by this "
                    "server. See --logits-processor-pattern engine argument "
275
276
                    "for more information."
                )
277
278
279
280
281
282
283
            try:
                logits_processor = resolve_obj_by_qualname(qualname)
            except Exception as e:
                raise ValueError(
                    f"Logits processor '{qualname}' could not be resolved: {e}"
                ) from e
            if isinstance(processor, LogitsProcessorConstructor):
284
285
286
                logits_processor = logits_processor(
                    *processor.args or [], **processor.kwargs or {}
                )
287
288
289
290
291
            logits_processors.append(logits_processor)
        return logits_processors
    elif processors:
        raise ValueError(
            "The `logits_processors` argument is not supported by this "
292
            "server. See --logits-processor-pattern engine argument "
293
294
            "for more information."
        )
295
296
297
    return None


298
ResponseInputOutputItem: TypeAlias = ResponseInputItemParam | ResponseOutputItem
299
300


301
302
303
class ResponsesRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/responses/create
304
305
    background: bool | None = False
    include: (
306
307
308
309
310
311
312
313
314
315
        list[
            Literal[
                "code_interpreter_call.outputs",
                "computer_call_output.output.image_url",
                "file_search_call.results",
                "message.input_image.image_url",
                "message.output_text.logprobs",
                "reasoning.encrypted_content",
            ],
        ]
316
317
318
319
320
321
322
323
        | None
    ) = None
    input: str | list[ResponseInputOutputItem]
    instructions: str | None = None
    max_output_tokens: int | None = None
    max_tool_calls: int | None = None
    metadata: Metadata | None = None
    model: str | None = None
324
    logit_bias: dict[str, float] | None = None
325
326
327
328
    parallel_tool_calls: bool | None = True
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
329
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] = "auto"
330
331
332
333
    store: bool | None = True
    stream: bool | None = False
    temperature: float | None = None
    text: ResponseTextConfig | None = None
334
335
    tool_choice: ToolChoice = "auto"
    tools: list[Tool] = Field(default_factory=list)
336
337
    top_logprobs: int | None = 0
    top_p: float | None = None
338
    top_k: int | None = None
339
340
    truncation: Literal["auto", "disabled"] | None = "disabled"
    user: str | None = None
341
342
343
344
345
346
347

    # --8<-- [start:responses-extra-params]
    request_id: str = Field(
        default_factory=lambda: f"resp_{random_uuid()}",
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
348
349
            "through out the inference process and return in response."
        ),
350
    )
351
    mm_processor_kwargs: dict[str, Any] | None = Field(
352
353
354
355
356
357
358
359
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
360
361
            "if the served model does not use priority scheduling."
        ),
362
    )
363
    cache_salt: str | None = Field(
364
365
366
367
368
369
370
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
371
            "to 256 bit)."
372
373
        ),
    )
374
375
376
377
378

    enable_response_messages: bool = Field(
        default=False,
        description=(
            "Dictates whether or not to return messages as part of the "
379
            "response object. Currently only supported for"
380
381
382
            "non-background and gpt-oss only. "
        ),
    )
383
384
385
386
387
    # similar to input_messages / output_messages in ResponsesResponse
    # we take in previous_input_messages (ie in harmony format)
    # this cannot be used in conjunction with previous_response_id
    # TODO: consider supporting non harmony messages as well
    previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None
388
389
390
391
392
    # --8<-- [end:responses-extra-params]

    _DEFAULT_SAMPLING_PARAMS = {
        "temperature": 1.0,
        "top_p": 1.0,
393
        "top_k": 0,
394
395
396
397
398
    }

    def to_sampling_params(
        self,
        default_max_tokens: int,
399
        default_sampling_params: dict | None = None,
400
401
402
403
404
405
406
407
408
    ) -> SamplingParams:
        if self.max_output_tokens is None:
            max_tokens = default_max_tokens
        else:
            max_tokens = min(self.max_output_tokens, default_max_tokens)

        default_sampling_params = default_sampling_params or {}
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
409
410
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
411
412
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
413
414
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
415
416
417
418
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
419
        stop_token_ids = default_sampling_params.get("stop_token_ids")
420
421

        # Structured output
422
        structured_outputs = None
423
424
        if self.text is not None and self.text.format is not None:
            response_format = self.text.format
425
426
427
428
            if (
                response_format.type == "json_schema"
                and response_format.schema_ is not None
            ):
429
                structured_outputs = StructuredOutputsParams(
430
431
                    json=response_format.schema_
                )
432
433
434
435
436
437
438
            elif response_format.type == "json_object":
                raise NotImplementedError("json_object is not supported")

        # TODO: add more parameters
        return SamplingParams.from_optional(
            temperature=temperature,
            top_p=top_p,
439
            top_k=top_k,
440
            max_tokens=max_tokens,
441
            logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
442
            stop_token_ids=stop_token_ids,
443
444
445
            output_kind=(
                RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
            ),
446
            structured_outputs=structured_outputs,
447
            logit_bias=self.logit_bias,
448
            skip_clone=True,  # Created fresh per request, safe to skip clone
449
450
        )

451
452
453
454
    def is_include_output_logprobs(self) -> bool:
        """Check if the request includes output logprobs."""
        if self.include is None:
            return False
455
456
457
458
        return (
            isinstance(self.include, list)
            and "message.output_text.logprobs" in self.include
        )
459

460
461
462
463
464
    @model_validator(mode="before")
    def validate_background(cls, data):
        if not data.get("background"):
            return data
        if not data.get("store", True):
465
            raise ValueError("background can only be used when `store` is true")
466
467
468
469
470
        return data

    @model_validator(mode="before")
    def validate_prompt(cls, data):
        if data.get("prompt") is not None:
471
472
473
            raise VLLMValidationError(
                "prompt template is not supported", parameter="prompt"
            )
474
475
        return data

476
477
    @model_validator(mode="before")
    def check_cache_salt_support(cls, data):
478
479
480
481
482
483
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
484
485
        return data

486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
    @model_validator(mode="before")
    def function_call_parsing(cls, data):
        """Parse function_call dictionaries into ResponseFunctionToolCall objects.
        This ensures Pydantic can properly resolve union types in the input field.
        Function calls provided as dicts are converted to ResponseFunctionToolCall
        objects before validation, while invalid structures are left for Pydantic
        to reject with appropriate error messages.
        """

        input_data = data.get("input")

        # Early return for None, strings, or bytes
        # (strings are iterable but shouldn't be processed)
        if input_data is None or isinstance(input_data, (str, bytes)):
            return data

        # Convert iterators (like ValidatorIterator) to list
        if not isinstance(input_data, list):
            try:
                input_data = list(input_data)
            except TypeError:
                # Not iterable, leave as-is for Pydantic to handle
                return data

        processed_input = []
        for item in input_data:
            if isinstance(item, dict) and item.get("type") == "function_call":
                try:
                    processed_input.append(ResponseFunctionToolCall(**item))
                except ValidationError:
                    # Let Pydantic handle validation for malformed function calls
                    logger.debug(
                        "Failed to parse function_call to ResponseFunctionToolCall, "
                        "leaving for Pydantic validation"
                    )
                    processed_input.append(item)
            else:
                processed_input.append(item)

        data["input"] = processed_input
        return data

528

529
class ChatCompletionRequest(OpenAIBaseModel):
530
531
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
532
    messages: list[ChatCompletionMessageParam]
533
534
535
536
537
538
    model: str | None = None
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: bool | None = False
    top_logprobs: int | None = 0
    max_tokens: int | None = Field(
539
        default=None,
540
541
        deprecated="max_tokens is deprecated in favor of "
        "the max_completion_tokens field",
542
    )
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
    max_completion_tokens: int | None = None
    n: int | None = 1
    presence_penalty: float | None = 0.0
    response_format: AnyResponseFormat | None = None
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    temperature: float | None = None
    top_p: float | None = None
    tools: list[ChatCompletionToolsParam] | None = None
    tool_choice: (
        Literal["none"]
        | Literal["auto"]
        | Literal["required"]
        | ChatCompletionNamedToolChoiceParam
        | None
    ) = "none"
    reasoning_effort: Literal["low", "medium", "high"] | None = None
562
    include_reasoning: bool = True
563
    parallel_tool_calls: bool | None = True
564

565
    # NOTE this will be ignored by vLLM
566
    user: str | None = None
567

568
    # --8<-- [start:chat-completion-sampling-params]
569
    use_beam_search: bool = False
570
571
572
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
573
    length_penalty: float = 1.0
574
    stop_token_ids: list[int] | None = []
575
576
577
578
579
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
580
581
582
    truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (
        None
    )
583
584
    prompt_logprobs: int | None = None
    allowed_token_ids: list[int] | None = None
585
    bad_words: list[str] = Field(default_factory=list)
586
    # --8<-- [end:chat-completion-sampling-params]
587

588
    # --8<-- [start:chat-completion-extra-params]
589
    echo: bool = Field(
590
591
592
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
593
594
            "if they belong to the same role."
        ),
595
    )
596
    add_generation_prompt: bool = Field(
597
        default=True,
598
599
600
601
602
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
603
    )
604
605
    continue_final_message: bool = Field(
        default=False,
606
607
608
609
610
611
612
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
613
    )
614
    add_special_tokens: bool = Field(
615
616
617
618
619
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
620
            "special tokens so this should be set to false (as is the "
621
622
            "default)."
        ),
623
    )
624
    documents: list[dict[str, str]] | None = Field(
625
        default=None,
626
627
628
629
630
631
632
        description=(
            "A list of dicts representing documents that will be accessible to "
            "the model if it is performing RAG (retrieval-augmented generation)."
            " If the template does not support RAG, this argument will have no "
            "effect. We recommend that each document should be a dict containing "
            '"title" and "text" keys.'
        ),
633
    )
634
    chat_template: str | None = Field(
635
636
637
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
638
639
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
640
641
            "does not define one."
        ),
642
    )
643
    chat_template_kwargs: dict[str, Any] | None = Field(
644
        default=None,
645
646
        description=(
            "Additional keyword args to pass to the template renderer. "
647
648
            "Will be accessible by the chat template."
        ),
649
    )
650
    mm_processor_kwargs: dict[str, Any] | None = Field(
651
652
653
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
654
    structured_outputs: StructuredOutputsParams | None = Field(
655
        default=None,
656
        description="Additional kwargs for structured outputs",
657
    )
658
659
660
661
662
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
663
664
            "if the served model does not use priority scheduling."
        ),
665
    )
666
    request_id: str = Field(
667
        default_factory=random_uuid,
668
669
670
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
671
672
            "through out the inference process and return in response."
        ),
673
    )
674
    logits_processors: LogitsProcessors | None = Field(
675
676
677
678
679
680
681
682
683
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
684
685
686
            "{'param': 'value'}}."
        ),
    )
687
    return_tokens_as_token_ids: bool | None = Field(
688
689
690
691
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
692
693
694
            "that are not JSON-encodable can be identified."
        ),
    )
695
    return_token_ids: bool | None = Field(
696
697
698
699
700
701
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
702
703
704
            "need to map generated text back to input tokens."
        ),
    )
705
    cache_salt: str | None = Field(
706
707
708
709
710
711
712
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
713
            "to 256 bit)."
714
715
        ),
    )
716
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
717
        default=None,
718
719
        description="KVTransfer parameters used for disaggregated serving.",
    )
720

721
    vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
722
        default=None,
723
        description=(
724
            "Additional request parameters with (list of) string or "
725
726
            "numeric values, used by custom extensions."
        ),
727
728
    )

729
    # --8<-- [end:chat-completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
730

731
732
733
734
735
    # Default sampling parameters for chat completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
736
        "top_k": 0,
737
738
739
740
        "min_p": 0.0,
    }

    def to_beam_search_params(
741
742
        self, max_tokens: int, default_sampling_params: dict
    ) -> BeamSearchParams:
743
        n = self.n if self.n is not None else 1
744
745
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
746
747
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
748
749
750
751
752
753

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
754
            length_penalty=self.length_penalty,
755
756
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
757

758
    def to_sampling_params(
759
        self,
760
        max_tokens: int,
761
        logits_processor_pattern: str | None,
762
        default_sampling_params: dict,
763
    ) -> SamplingParams:
764
765
766
767
768
769
770
771
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
772
773
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
774
775
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
776
777
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
778
779
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
780
781
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
782
783
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
784
785
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
786

787
788
789
790
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.top_logprobs

791
        response_format = self.response_format
792
        if response_format is not None:
793
794
795
796
797
798
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
            if response_format.type == "json_object":
                self.structured_outputs.json_object = True
            elif response_format.type == "json_schema":
                json_schema = response_format.json_schema
                assert json_schema is not None
                self.structured_outputs.json = json_schema.json_schema
            elif response_format.type == "structural_tag":
                structural_tag = response_format
                assert structural_tag is not None and isinstance(
                    structural_tag,
                    (
                        LegacyStructuralTagResponseFormat,
                        StructuralTagResponseFormat,
                    ),
                )
                s_tag_obj = structural_tag.model_dump(by_alias=True)
                self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
816

817
818
819
820
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
821
        return SamplingParams.from_optional(
822
823
824
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
825
826
827
828
829
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
830
            seed=self.seed,
831
832
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
833
            logprobs=self.top_logprobs if self.logprobs else None,
834
            prompt_logprobs=prompt_logprobs,
835
            ignore_eos=self.ignore_eos,
836
            max_tokens=max_tokens,
837
            min_tokens=self.min_tokens,
838
839
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
840
841
842
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
843
            include_stop_str_in_output=self.include_stop_str_in_output,
844
            truncate_prompt_tokens=self.truncate_prompt_tokens,
845
846
847
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
848
            structured_outputs=self.structured_outputs,
Robert Shaw's avatar
Robert Shaw committed
849
            logit_bias=self.logit_bias,
850
            bad_words=self.bad_words,
851
            allowed_token_ids=self.allowed_token_ids,
852
            extra_args=extra_args or None,
853
            skip_clone=True,  # Created fresh per request, safe to skip clone
854
        )
855

856
    @model_validator(mode="before")
857
    @classmethod
858
859
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
860
861
862
863
            raise VLLMValidationError(
                "Stream options can only be defined when `stream=True`.",
                parameter="stream_options",
            )
864
865
866
867
868
869
870

        return data

    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
871
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
872
873
874
                raise VLLMValidationError(
                    "`prompt_logprobs` are not available when `stream=True`.",
                    parameter="prompt_logprobs",
875
                )
876

877
            if prompt_logprobs < 0 and prompt_logprobs != -1:
878
879
880
881
882
                raise VLLMValidationError(
                    "`prompt_logprobs` must be a positive value or -1.",
                    parameter="prompt_logprobs",
                    value=prompt_logprobs,
                )
883
        if (top_logprobs := data.get("top_logprobs")) is not None:
884
            if top_logprobs < 0 and top_logprobs != -1:
885
886
887
888
889
                raise VLLMValidationError(
                    "`top_logprobs` must be a positive value or -1.",
                    parameter="top_logprobs",
                    value=top_logprobs,
                )
890

891
            if (top_logprobs == -1 or top_logprobs > 0) and not data.get("logprobs"):
892
893
894
                raise VLLMValidationError(
                    "when using `top_logprobs`, `logprobs` must be set to true.",
                    parameter="top_logprobs",
895
896
897
                )

        return data
898

899
900
    @model_validator(mode="before")
    @classmethod
901
    def check_structured_outputs_count(cls, data):
902
903
904
        if isinstance(data, ValueError):
            raise data

905
        if data.get("structured_outputs", None) is None:
906
907
            return data

908
        structured_outputs_kwargs = data["structured_outputs"]
909
910
        count = sum(
            structured_outputs_kwargs.get(k) is not None
911
912
            for k in ("json", "regex", "choice")
        )
913
914
        # you can only use one kind of constraints for structured outputs
        if count > 1:
915
            raise ValueError(
916
                "You can only use one kind of constraints for structured "
917
918
                "outputs ('json', 'regex' or 'choice')."
            )
919
920
        # you can only either use structured outputs or tools, not both
        if count > 1 and data.get("tool_choice", "none") not in (
921
922
923
            "none",
            "auto",
            "required",
924
        ):
925
            raise ValueError(
926
                "You can only either use constraints for structured outputs "
927
928
                "or tools, not both."
            )
929
930
931
932
        return data

    @model_validator(mode="before")
    @classmethod
933
934
935
    def check_tool_usage(cls, data):
        # if "tool_choice" is not specified but tools are provided,
        # default to "auto" tool_choice
936
        if "tool_choice" not in data and data.get("tools"):
937
938
            data["tool_choice"] = "auto"

939
        # if "tool_choice" is "none" -- no validation is needed for tools
940
941
942
        if "tool_choice" in data and data["tool_choice"] == "none":
            return data

943
        # if "tool_choice" is specified -- validation
944
        if "tool_choice" in data and data["tool_choice"] is not None:
945
            # ensure that if "tool choice" is specified, tools are present
946
            if "tools" not in data or data["tools"] is None:
947
                raise ValueError("When using `tool_choice`, `tools` must be set.")
948
949

            # make sure that tool choice is either a named tool
950
            # OR that it's set to "auto" or "required"
951
952
953
            if data["tool_choice"] not in ["auto", "required"] and not isinstance(
                data["tool_choice"], dict
            ):
954
                raise ValueError(
955
956
957
                    f"Invalid value for `tool_choice`: {data['tool_choice']}! "
                    'Only named tools, "none", "auto" or "required" '
                    "are supported."
958
                )
959

960
961
962
            # if tool_choice is "required" but the "tools" list is empty,
            # override the data to behave like "none" to align with
            # OpenAI’s behavior.
963
964
965
966
967
            if (
                data["tool_choice"] == "required"
                and isinstance(data["tools"], list)
                and len(data["tools"]) == 0
            ):
968
969
970
971
                data["tool_choice"] = "none"
                del data["tools"]
                return data

972
973
            # ensure that if "tool_choice" is specified as an object,
            # it matches a valid tool
974
975
            correct_usage_message = (
                'Correct usage: `{"type": "function",'
976
                ' "function": {"name": "my_function"}}`'
977
            )
978
979
            if isinstance(data["tool_choice"], dict):
                valid_tool = False
980
981
                function = data["tool_choice"].get("function")
                if not isinstance(function, dict):
982
                    raise ValueError(
983
                        f"Invalid value for `function`: `{function}` in "
984
985
                        f"`tool_choice`! {correct_usage_message}"
                    )
986
                if "name" not in function:
987
988
989
990
                    raise ValueError(
                        f"Expected field `name` in `function` in "
                        f"`tool_choice`! {correct_usage_message}"
                    )
991
                function_name = function["name"]
992
                if not isinstance(function_name, str) or len(function_name) == 0:
993
                    raise ValueError(
994
                        f"Invalid `name` in `function`: `{function_name}`"
995
996
                        f" in `tool_choice`! {correct_usage_message}"
                    )
997
                for tool in data["tools"]:
998
                    if tool["function"]["name"] == function_name:
999
1000
1001
1002
1003
                        valid_tool = True
                        break
                if not valid_tool:
                    raise ValueError(
                        "The tool specified in `tool_choice` does not match any"
1004
1005
                        " of the specified `tools`"
                    )
1006
1007
        return data

1008
1009
1010
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
1011
1012
1013
1014
1015
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
1016
1017
        return data

1018
1019
1020
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
1021
1022
1023
1024
1025
1026
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
1027
1028
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1029

1030
class CompletionRequest(OpenAIBaseModel):
1031
1032
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
1033
1034
1035
1036
1037
1038
1039
    model: str | None = None
    prompt: list[int] | list[list[int]] | str | list[str] | None = None
    echo: bool | None = False
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: int | None = None
    max_tokens: int | None = 16
1040
    n: int = 1
1041
1042
1043
1044
1045
1046
1047
1048
1049
    presence_penalty: float | None = 0.0
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    suffix: str | None = None
    temperature: float | None = None
    top_p: float | None = None
    user: str | None = None
1050

1051
    # --8<-- [start:completion-sampling-params]
1052
    use_beam_search: bool = False
1053
1054
1055
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
1056
    length_penalty: float = 1.0
1057
    stop_token_ids: list[int] | None = []
1058
1059
1060
1061
1062
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
1063
1064
1065
    truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (
        None
    )
1066
1067
    allowed_token_ids: list[int] | None = None
    prompt_logprobs: int | None = None
1068
    # --8<-- [end:completion-sampling-params]
1069

1070
    # --8<-- [start:completion-extra-params]
1071
    prompt_embeds: bytes | list[bytes] | None = None
1072
1073
    add_special_tokens: bool = Field(
        default=True,
1074
        description=(
1075
            "If true (the default), special tokens (e.g. BOS) will be added to "
1076
1077
            "the prompt."
        ),
1078
    )
1079
    response_format: AnyResponseFormat | None = Field(
1080
        default=None,
1081
1082
1083
1084
1085
        description=(
            "Similar to chat completion, this parameter specifies the format "
            "of output. Only {'type': 'json_object'}, {'type': 'json_schema'}"
            ", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
        ),
1086
    )
1087
    structured_outputs: StructuredOutputsParams | None = Field(
1088
        default=None,
1089
        description="Additional kwargs for structured outputs",
1090
    )
1091
1092
1093
1094
1095
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
1096
1097
            "if the served model does not use priority scheduling."
        ),
1098
    )
1099
    request_id: str = Field(
1100
        default_factory=random_uuid,
1101
1102
1103
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
1104
1105
            "through out the inference process and return in response."
        ),
1106
    )
1107
    logits_processors: LogitsProcessors | None = Field(
1108
1109
1110
1111
1112
1113
1114
1115
1116
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
1117
1118
1119
            "{'param': 'value'}}."
        ),
    )
1120

1121
    return_tokens_as_token_ids: bool | None = Field(
1122
1123
1124
1125
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
1126
1127
1128
            "that are not JSON-encodable can be identified."
        ),
    )
1129
    return_token_ids: bool | None = Field(
1130
1131
1132
1133
1134
1135
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
1136
1137
1138
            "need to map generated text back to input tokens."
        ),
    )
1139

1140
    cache_salt: str | None = Field(
1141
1142
1143
1144
1145
1146
1147
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
1148
            "to 256 bit)."
1149
1150
        ),
    )
1151

1152
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
1153
        default=None,
1154
1155
        description="KVTransfer parameters used for disaggregated serving.",
    )
Robert Shaw's avatar
Robert Shaw committed
1156

1157
    vllm_xargs: dict[str, str | int | float] | None = Field(
1158
        default=None,
1159
1160
1161
1162
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
1163
1164
    )

1165
    # --8<-- [end:completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
1166

1167
1168
1169
1170
1171
    # Default sampling parameters for completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
1172
        "top_k": 0,
1173
1174
1175
1176
        "min_p": 0.0,
    }

    def to_beam_search_params(
1177
1178
        self,
        max_tokens: int,
1179
        default_sampling_params: dict | None = None,
1180
1181
1182
    ) -> BeamSearchParams:
        if default_sampling_params is None:
            default_sampling_params = {}
1183
        n = self.n if self.n is not None else 1
1184
1185
1186

        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get("temperature", 1.0)
1187
1188
1189
1190
1191
1192

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
1193
            length_penalty=self.length_penalty,
1194
1195
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
1196

1197
    def to_sampling_params(
1198
        self,
1199
        max_tokens: int,
1200
1201
        logits_processor_pattern: str | None,
        default_sampling_params: dict | None = None,
1202
    ) -> SamplingParams:
1203
1204
        if default_sampling_params is None:
            default_sampling_params = {}
1205

1206
1207
1208
1209
1210
1211
1212
1213
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
1214
1215
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
1216
1217
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
1218
1219
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
1220
1221
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
1222
1223
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
1224
1225
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
1226
1227
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
1228

1229
1230
1231
1232
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

1233
1234
        echo_without_generation = self.echo and self.max_tokens == 0

1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
        response_format = self.response_format
        if response_format is not None:
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
            if response_format.type == "json_object":
                self.structured_outputs.json_object = True
            elif response_format.type == "json_schema":
                json_schema = response_format.json_schema
1247
                assert json_schema is not None
1248
1249
1250
                self.structured_outputs.json = json_schema.json_schema
            elif response_format.type == "structural_tag":
                structural_tag = response_format
1251
                assert structural_tag is not None and isinstance(
1252
1253
1254
1255
1256
                    structural_tag,
                    (
                        LegacyStructuralTagResponseFormat,
                        StructuralTagResponseFormat,
                    ),
1257
1258
                )
                s_tag_obj = structural_tag.model_dump(by_alias=True)
1259
                self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
1260

1261
1262
1263
1264
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
1265
        return SamplingParams.from_optional(
1266
1267
1268
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
1269
1270
1271
1272
1273
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
1274
            seed=self.seed,
1275
1276
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
1277
            logprobs=self.logprobs,
1278
            ignore_eos=self.ignore_eos,
1279
            max_tokens=max_tokens if not echo_without_generation else 1,
1280
            min_tokens=self.min_tokens,
1281
            prompt_logprobs=prompt_logprobs,
1282
            skip_special_tokens=self.skip_special_tokens,
1283
            spaces_between_special_tokens=self.spaces_between_special_tokens,
1284
            include_stop_str_in_output=self.include_stop_str_in_output,
1285
1286
1287
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
1288
            truncate_prompt_tokens=self.truncate_prompt_tokens,
1289
1290
1291
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
1292
            structured_outputs=self.structured_outputs,
1293
            logit_bias=self.logit_bias,
Robert Shaw's avatar
Robert Shaw committed
1294
            allowed_token_ids=self.allowed_token_ids,
1295
            extra_args=extra_args or None,
1296
            skip_clone=True,  # Created fresh per request, safe to skip clone
1297
        )
1298

1299
1300
    @model_validator(mode="before")
    @classmethod
1301
    def check_structured_outputs_count(cls, data):
1302
        if data.get("structured_outputs", None) is None:
1303
1304
            return data

1305
        structured_outputs_kwargs = data["structured_outputs"]
1306
1307
        count = sum(
            structured_outputs_kwargs.get(k) is not None
1308
1309
            for k in ("json", "regex", "choice")
        )
1310
        if count > 1:
1311
            raise VLLMValidationError(
1312
                "You can only use one kind of constraints for structured "
1313
1314
                "outputs ('json', 'regex' or 'choice').",
                parameter="structured_outputs",
1315
            )
1316
1317
        return data

1318
1319
1320
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
1321
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
1322
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
1323
1324
1325
                raise VLLMValidationError(
                    "`prompt_logprobs` are not available when `stream=True`.",
                    parameter="prompt_logprobs",
1326
                )
1327

1328
            if prompt_logprobs < 0 and prompt_logprobs != -1:
1329
1330
1331
1332
1333
                raise VLLMValidationError(
                    "`prompt_logprobs` must be a positive value or -1.",
                    parameter="prompt_logprobs",
                    value=prompt_logprobs,
                )
1334
        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
1335
1336
1337
1338
1339
            raise VLLMValidationError(
                "`logprobs` must be a positive value.",
                parameter="logprobs",
                value=logprobs,
            )
1340

1341
1342
        return data

1343
1344
1345
1346
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
1347
1348
1349
1350
            raise VLLMValidationError(
                "Stream options can only be defined when `stream=True`.",
                parameter="stream_options",
            )
1351

1352
1353
        return data

1354
1355
1356
    @model_validator(mode="before")
    @classmethod
    def validate_prompt_and_prompt_embeds(cls, data):
1357
1358
1359
        prompt = data.get("prompt")
        prompt_embeds = data.get("prompt_embeds")

1360
1361
1362
1363
        prompt_is_empty = prompt is None or (isinstance(prompt, str) and prompt == "")
        embeds_is_empty = prompt_embeds is None or (
            isinstance(prompt_embeds, list) and len(prompt_embeds) == 0
        )
1364
1365

        if prompt_is_empty and embeds_is_empty:
1366
            raise ValueError(
1367
1368
1369
                "Either prompt or prompt_embeds must be provided and non-empty."
            )

1370
1371
        return data

1372
1373
1374
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
1375
1376
1377
1378
1379
1380
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
1381
1382
        return data

Zhuohan Li's avatar
Zhuohan Li committed
1383

1384
class CompletionLogProbs(OpenAIBaseModel):
1385
    text_offset: list[int] = Field(default_factory=list)
1386
    token_logprobs: list[float | None] = Field(default_factory=list)
1387
    tokens: list[str] = Field(default_factory=list)
1388
    top_logprobs: list[dict[str, float] | None] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
1389
1390


1391
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1392
1393
    index: int
    text: str
1394
1395
1396
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1397
1398
1399
1400
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1401
1402
            "including encountering the EOS token"
        ),
1403
    )
1404
1405
1406
    token_ids: list[int] | None = None  # For response
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None  # For prompt
Zhuohan Li's avatar
Zhuohan Li committed
1407
1408


1409
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1410
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
1411
    object: Literal["text_completion"] = "text_completion"
Zhuohan Li's avatar
Zhuohan Li committed
1412
1413
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1414
    choices: list[CompletionResponseChoice]
1415
1416
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1417
    usage: UsageInfo
1418
1419

    # vLLM-specific fields that are not in OpenAI spec
1420
    kv_transfer_params: dict[str, Any] | None = Field(
1421
1422
        default=None, description="KVTransfer parameters."
    )
Zhuohan Li's avatar
Zhuohan Li committed
1423
1424


1425
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1426
1427
    index: int
    text: str
1428
1429
1430
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
1431
1432
1433
1434
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
1435
1436
            "including encountering the EOS token"
        ),
1437
    )
1438
1439
    # not part of the OpenAI spec but for tracing the tokens
    # prompt tokens is put into choice to align with CompletionResponseChoice
1440
1441
    prompt_token_ids: list[int] | None = None
    token_ids: list[int] | None = None
Zhuohan Li's avatar
Zhuohan Li committed
1442
1443


1444
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
1445
1446
1447
1448
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1449
    choices: list[CompletionResponseStreamChoice]
1450
    usage: UsageInfo | None = Field(default=None)
1451
1452


1453
1454
1455
1456
1457
1458
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
1459
    id: str = Field(default_factory=make_tool_call_id)
1460
1461
1462
1463
    type: Literal["function"] = "function"
    function: FunctionCall


1464
class DeltaFunctionCall(BaseModel):
1465
1466
    name: str | None = None
    arguments: str | None = None
1467
1468
1469
1470


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
1471
1472
    id: str | None = None
    type: Literal["function"] | None = None
1473
    index: int
1474
    function: DeltaFunctionCall | None = None
1475
1476
1477
1478
1479
1480
1481


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
1482
    tool_calls: list[ToolCall]
1483
1484
1485

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
1486
    content: str | None = None
1487
1488


1489
class ChatMessage(OpenAIBaseModel):
1490
    role: str
1491
1492
1493
1494
1495
    content: str | None = None
    refusal: str | None = None
    annotations: OpenAIAnnotation | None = None
    audio: OpenAIChatCompletionAudio | None = None
    function_call: FunctionCall | None = None
1496
    tool_calls: list[ToolCall] = Field(default_factory=list)
1497

1498
    # vLLM-specific fields that are not in OpenAI spec
1499
    reasoning: str | None = None
1500
    reasoning_content: str | None = None
1501
1502
1503
1504
1505
1506
1507
    """Deprecated: use `reasoning` instead."""

    @model_validator(mode="after")
    def handle_deprecated_reasoning_content(self):
        """Copy reasoning to reasoning_content for backward compatibility."""
        self.reasoning_content = self.reasoning
        return self
1508

1509

1510
1511
1512
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
1513
    bytes: list[int] | None = None
1514
1515
1516


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
1517
1518
    # Workaround: redefine fields name cache so that it's not
    # shared with the super class.
1519
    field_names: ClassVar[set[str] | None] = None
1520
    top_logprobs: list[ChatCompletionLogProb] = Field(default_factory=list)
1521
1522
1523


class ChatCompletionLogProbs(OpenAIBaseModel):
1524
    content: list[ChatCompletionLogProbsContent] | None = None
1525
1526


1527
class ChatCompletionResponseChoice(OpenAIBaseModel):
1528
1529
    index: int
    message: ChatMessage
1530
    logprobs: ChatCompletionLogProbs | None = None
1531
    # per OpenAI spec this is the default
1532
    finish_reason: str | None = "stop"
1533
    # not part of the OpenAI spec but included in vLLM for legacy reasons
1534
    stop_reason: int | str | None = None
1535
1536
    # not part of the OpenAI spec but is useful for tracing the tokens
    # in agent scenarios
1537
    token_ids: list[int] | None = None
1538
1539


1540
class ChatCompletionResponse(OpenAIBaseModel):
1541
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
1542
    object: Literal["chat.completion"] = "chat.completion"
1543
1544
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1545
    choices: list[ChatCompletionResponseChoice]
1546
1547
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
1548
    usage: UsageInfo
1549
1550

    # vLLM-specific fields that are not in OpenAI spec
1551
1552
1553
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None
    kv_transfer_params: dict[str, Any] | None = Field(
1554
1555
        default=None, description="KVTransfer parameters."
    )
1556
1557


1558
class DeltaMessage(OpenAIBaseModel):
1559
1560
    role: str | None = None
    content: str | None = None
1561
    reasoning: str | None = None
1562
    reasoning_content: str | None = None
1563
    """Deprecated: use `reasoning` instead."""
1564
    tool_calls: list[DeltaToolCall] = Field(default_factory=list)
1565

1566
1567
1568
1569
1570
1571
    @model_validator(mode="after")
    def handle_deprecated_reasoning_content(self):
        """Copy reasoning to reasoning_content for backward compatibility."""
        self.reasoning_content = self.reasoning
        return self

1572

1573
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
1574
1575
    index: int
    delta: DeltaMessage
1576
1577
1578
    logprobs: ChatCompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = None
1579
    # not part of the OpenAI spec but for tracing the tokens
1580
    token_ids: list[int] | None = None
1581
1582


1583
class ChatCompletionStreamResponse(OpenAIBaseModel):
1584
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
1585
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
1586
1587
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
1588
    choices: list[ChatCompletionResponseStreamChoice]
1589
    usage: UsageInfo | None = Field(default=None)
1590
    # not part of the OpenAI spec but for tracing the tokens
1591
    prompt_token_ids: list[int] | None = None
1592
1593


1594
1595
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
1596
1597
    finish_reason: str | None = None
    stop_reason: int | str | None = None
1598
1599
1600
1601
1602
1603
1604
1605


class TranscriptionStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
    object: Literal["transcription.chunk"] = "transcription.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranscriptionResponseStreamChoice]
1606
    usage: UsageInfo | None = Field(default=None)
1607
1608


1609
1610
class InputTokensDetails(OpenAIBaseModel):
    cached_tokens: int
1611
1612
    input_tokens_per_turn: list[int] = Field(default_factory=list)
    cached_tokens_per_turn: list[int] = Field(default_factory=list)
1613
1614
1615


class OutputTokensDetails(OpenAIBaseModel):
1616
1617
    reasoning_tokens: int = 0
    tool_output_tokens: int = 0
1618
1619
    output_tokens_per_turn: list[int] = Field(default_factory=list)
    tool_output_tokens_per_turn: list[int] = Field(default_factory=list)
1620
1621
1622
1623
1624
1625
1626
1627


class ResponseUsage(OpenAIBaseModel):
    input_tokens: int
    input_tokens_details: InputTokensDetails
    output_tokens: int
    output_tokens_details: OutputTokensDetails
    total_tokens: int
1628
1629


1630
1631
1632
1633
1634
1635
def serialize_message(msg):
    """
    Serializes a single message
    """
    if isinstance(msg, dict):
        return msg
1636
    elif hasattr(msg, "to_dict"):
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
        return msg.to_dict()
    else:
        # fallback to pyandic dump
        return msg.model_dump_json()


def serialize_messages(msgs):
    """
    Serializes multiple messages
    """
    return [serialize_message(msg) for msg in msgs] if msgs else None


1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
class ResponseRawMessageAndToken(OpenAIBaseModel):
    """Class to show the raw message.
    If message / tokens diverge, tokens is the source of truth"""

    message: str
    tokens: list[int]
    type: Literal["raw_message_tokens"] = "raw_message_tokens"


ResponseInputOutputMessage: TypeAlias = (
    list[ChatCompletionMessageParam] | list[ResponseRawMessageAndToken]
)


1664
1665
1666
1667
class ResponsesResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"resp_{random_uuid()}")
    created_at: int = Field(default_factory=lambda: int(time.time()))
    # error: Optional[ResponseError] = None
1668
1669
1670
    incomplete_details: IncompleteDetails | None = None
    instructions: str | None = None
    metadata: Metadata | None = None
1671
1672
    model: str
    object: Literal["response"] = "response"
1673
    output: list[ResponseOutputItem]
1674
1675
1676
1677
1678
1679
1680
    parallel_tool_calls: bool
    temperature: float
    tool_choice: ToolChoice
    tools: list[Tool]
    top_p: float
    background: bool
    max_output_tokens: int
1681
1682
1683
1684
    max_tool_calls: int | None = None
    previous_response_id: str | None = None
    prompt: ResponsePrompt | None = None
    reasoning: Reasoning | None = None
1685
1686
    service_tier: Literal["auto", "default", "flex", "scale", "priority"]
    status: ResponseStatus
1687
1688
    text: ResponseTextConfig | None = None
    top_logprobs: int | None = None
1689
    truncation: Literal["auto", "disabled"]
1690
1691
    usage: ResponseUsage | None = None
    user: str | None = None
1692

1693
    # --8<-- [start:responses-response-extra-params]
1694
1695
1696
    # These are populated when enable_response_messages is set to True
    # NOTE: custom serialization is needed
    # see serialize_input_messages and serialize_output_messages
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
    input_messages: ResponseInputOutputMessage | None = Field(
        default=None,
        description=(
            "If enable_response_messages, we can show raw token input to model."
        ),
    )
    output_messages: ResponseInputOutputMessage | None = Field(
        default=None,
        description=(
            "If enable_response_messages, we can show raw token output of model."
        ),
    )
    # --8<-- [end:responses-response-extra-params]
1710
1711
1712
1713
1714
1715

    # NOTE: openAI harmony doesn't serialize TextContent properly,
    # TODO: this fixes for TextContent, but need to verify for tools etc
    # https://github.com/openai/harmony/issues/78
    @field_serializer("output_messages", when_used="json")
    def serialize_output_messages(self, msgs, _info):
1716
        return serialize_messages(msgs)
1717
1718
1719
1720
1721

    # NOTE: openAI harmony doesn't serialize TextContent properly, this fixes it
    # https://github.com/openai/harmony/issues/78
    @field_serializer("input_messages", when_used="json")
    def serialize_input_messages(self, msgs, _info):
1722
        return serialize_messages(msgs)
1723

1724
1725
1726
1727
1728
1729
1730
1731
1732
    @classmethod
    def from_request(
        cls,
        request: ResponsesRequest,
        sampling_params: SamplingParams,
        model_name: str,
        created_time: int,
        output: list[ResponseOutputItem],
        status: ResponseStatus,
1733
        usage: ResponseUsage | None = None,
1734
1735
        input_messages: ResponseInputOutputMessage | None = None,
        output_messages: ResponseInputOutputMessage | None = None,
1736
    ) -> "ResponsesResponse":
1737
        incomplete_details: IncompleteDetails | None = None
1738
1739
        if status == "incomplete":
            incomplete_details = IncompleteDetails(reason="max_output_tokens")
1740
1741
1742
        # TODO: implement the other reason for incomplete_details,
        # which is content_filter
        # incomplete_details = IncompleteDetails(reason='content_filter')
1743
1744
1745
        return cls(
            id=request.request_id,
            created_at=created_time,
1746
            incomplete_details=incomplete_details,
1747
1748
1749
1750
            instructions=request.instructions,
            metadata=request.metadata,
            model=model_name,
            output=output,
1751
1752
            input_messages=input_messages,
            output_messages=output_messages,
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
            parallel_tool_calls=request.parallel_tool_calls,
            temperature=sampling_params.temperature,
            tool_choice=request.tool_choice,
            tools=request.tools,
            top_p=sampling_params.top_p,
            background=request.background,
            max_output_tokens=sampling_params.max_tokens,
            max_tool_calls=request.max_tool_calls,
            previous_response_id=request.previous_response_id,
            prompt=request.prompt,
            reasoning=request.reasoning,
            service_tier=request.service_tier,
            status=status,
            text=request.text,
            top_logprobs=sampling_params.logprobs,
            truncation=request.truncation,
            user=request.user,
            usage=usage,
        )


1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartDoneEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.done"]
    """The type of the event. Always `response.reasoning_part.done`."""


# TODO: this code can be removed once
# https://github.com/openai/openai-python/issues/2634 has been resolved
class ResponseReasoningPartAddedEvent(OpenAIBaseModel):
    content_index: int
    """The index of the content part that is done."""

    item_id: str
    """The ID of the output item that the content part was added to."""

    output_index: int
    """The index of the output item that the content part was added to."""

    part: ResponseReasoningTextContent
    """The content part that is done."""

    sequence_number: int
    """The sequence number of this event."""

    type: Literal["response.reasoning_part.added"]
    """The type of the event. Always `response.reasoning_part.added`."""


1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
# vLLM Streaming Events
# Note: we override the response type with the vLLM ResponsesResponse type
class ResponseCompletedEvent(OpenAIResponseCompletedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseCreatedEvent(OpenAIResponseCreatedEvent):
    response: ResponsesResponse  # type: ignore[override]


class ResponseInProgressEvent(OpenAIResponseInProgressEvent):
    response: ResponsesResponse  # type: ignore[override]


1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
StreamingResponsesResponse: TypeAlias = (
    ResponseCreatedEvent
    | ResponseInProgressEvent
    | ResponseCompletedEvent
    | ResponseOutputItemAddedEvent
    | ResponseOutputItemDoneEvent
    | ResponseContentPartAddedEvent
    | ResponseContentPartDoneEvent
    | ResponseReasoningTextDeltaEvent
    | ResponseReasoningTextDoneEvent
    | ResponseReasoningPartAddedEvent
    | ResponseReasoningPartDoneEvent
    | ResponseCodeInterpreterCallInProgressEvent
    | ResponseCodeInterpreterCallCodeDeltaEvent
    | ResponseWebSearchCallInProgressEvent
    | ResponseWebSearchCallSearchingEvent
    | ResponseWebSearchCallCompletedEvent
    | ResponseCodeInterpreterCallCodeDoneEvent
    | ResponseCodeInterpreterCallInterpretingEvent
    | ResponseCodeInterpreterCallCompletedEvent
1852
1853
1854
1855
    | ResponseMcpCallArgumentsDeltaEvent
    | ResponseMcpCallArgumentsDoneEvent
    | ResponseMcpCallInProgressEvent
    | ResponseMcpCallCompletedEvent
1856
)
1857

1858

1859
class TokenizeCompletionRequest(OpenAIBaseModel):
1860
    model: str | None = None
1861
1862
    prompt: str

1863
1864
1865
1866
    add_special_tokens: bool = Field(
        default=True,
        description=(
            "If true (the default), special tokens (e.g. BOS) will be added to "
1867
1868
            "the prompt."
        ),
1869
    )
1870
    return_token_strs: bool | None = Field(
1871
        default=False,
1872
1873
1874
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
1875
    )
1876
1877
1878


class TokenizeChatRequest(OpenAIBaseModel):
1879
    model: str | None = None
1880
    messages: list[ChatCompletionMessageParam]
1881

1882
1883
    add_generation_prompt: bool = Field(
        default=True,
1884
1885
1886
1887
1888
        description=(
            "If true, the generation prompt will be added to the chat template. "
            "This is a parameter used by chat template in tokenizer config of the "
            "model."
        ),
1889
    )
1890
    return_token_strs: bool | None = Field(
1891
        default=False,
1892
1893
1894
        description=(
            "If true, also return the token strings corresponding to the token ids."
        ),
1895
    )
1896
1897
    continue_final_message: bool = Field(
        default=False,
1898
1899
1900
1901
1902
1903
1904
        description=(
            "If this is set, the chat will be formatted so that the final "
            "message in the chat is open-ended, without any EOS tokens. The "
            "model will continue this message rather than starting a new one. "
            'This allows you to "prefill" part of the model\'s response for it. '
            "Cannot be used at the same time as `add_generation_prompt`."
        ),
1905
1906
1907
1908
1909
1910
1911
1912
    )
    add_special_tokens: bool = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to false (as is the "
1913
1914
            "default)."
        ),
1915
    )
1916
    chat_template: str | None = Field(
1917
1918
1919
1920
1921
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
1922
1923
            "does not define one."
        ),
1924
    )
1925
    chat_template_kwargs: dict[str, Any] | None = Field(
1926
        default=None,
1927
1928
        description=(
            "Additional keyword args to pass to the template renderer. "
1929
1930
            "Will be accessible by the chat template."
        ),
1931
    )
1932
    mm_processor_kwargs: dict[str, Any] | None = Field(
1933
1934
1935
        default=None,
        description=("Additional kwargs to pass to the HF processor."),
    )
1936
    tools: list[ChatCompletionToolsParam] | None = Field(
1937
1938
1939
        default=None,
        description=("A list of tools the model may call."),
    )
1940

1941
1942
1943
    @model_validator(mode="before")
    @classmethod
    def check_generation_prompt(cls, data):
1944
1945
1946
1947
1948
        if data.get("continue_final_message") and data.get("add_generation_prompt"):
            raise ValueError(
                "Cannot set both `continue_final_message` and "
                "`add_generation_prompt` to True."
            )
1949
1950
        return data

1951

1952
TokenizeRequest: TypeAlias = TokenizeCompletionRequest | TokenizeChatRequest
1953
1954
1955
1956
1957


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
1958
    tokens: list[int]
1959
    token_strs: list[str] | None = None
1960
1961
1962


class DetokenizeRequest(OpenAIBaseModel):
1963
    model: str | None = None
1964
    tokens: list[int]
1965
1966
1967
1968


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str
1969
1970


1971
1972
class TokenizerInfoResponse(OpenAIBaseModel):
    """
1973
    Response containing tokenizer configuration
1974
1975
1976
1977
1978
1979
1980
    equivalent to tokenizer_config.json
    """

    model_config = ConfigDict(extra="allow")
    tokenizer_class: str


1981
class LoadLoRAAdapterRequest(BaseModel):
1982
1983
1984
1985
    lora_name: str
    lora_path: str


1986
class UnloadLoRAAdapterRequest(BaseModel):
1987
    lora_name: str
1988
    lora_int_id: int | None = Field(default=None)
1989
1990
1991


## Protocols for Audio
1992
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
1993
1994
1995
1996


class TranscriptionRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
1997
    # https://platform.openai.com/docs/api-reference/audio/createTranscription
1998
1999
2000
2001
2002
2003
2004

    file: UploadFile
    """
    The audio file object (not file name) to transcribe, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

2005
    model: str | None = None
2006
2007
2008
    """ID of the model to use.
    """

2009
    language: str | None = None
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
    """The language of the input audio.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy and latency.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    ## TODO (varun) : Support if set to 0, certain thresholds are met !!

2033
    timestamp_granularities: list[Literal["word", "segment"]] = Field(
2034
2035
        alias="timestamp_granularities[]", default=[]
    )
2036
2037
2038
2039
2040
2041
2042
2043
    """The timestamp granularities to populate for this transcription.

    `response_format` must be set `verbose_json` to use timestamp granularities.
    Either or both of these options are supported: `word`, or `segment`. Note:
    There is no additional latency for segment timestamps, but generating word
    timestamps incurs additional latency.
    """

2044
    stream: bool | None = False
2045
    """When set, it will enable output to be streamed in a similar fashion
2046
    as the Chat Completion endpoint.
2047
    """
2048
    # --8<-- [start:transcription-extra-params]
2049
    # Flattened stream option to simplify form data.
2050
2051
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2052

2053
    vllm_xargs: dict[str, str | int | float] | None = Field(
2054
        default=None,
2055
2056
2057
2058
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
2059
    )
2060
    # --8<-- [end:transcription-extra-params]
2061

2062
    to_language: str | None = None
2063
2064
    """The language of the output audio we transcribe to.

2065
    Please note that this is not currently used by supported models at this
2066
2067
2068
    time, but it is a placeholder for future use, matching translation api.
    """

2069
    # --8<-- [start:transcription-sampling-params]
2070
2071
2072
2073
2074
2075
2076
2077
2078
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """

2079
    top_p: float | None = None
2080
    """Enables nucleus (top-p) sampling, where tokens are selected from the
2081
2082
2083
    smallest possible set whose cumulative probability exceeds `p`.
    """

2084
    top_k: int | None = None
2085
2086
    """Limits sampling to the `k` most probable tokens at each step."""

2087
    min_p: float | None = None
2088
    """Filters out tokens with a probability lower than `min_p`, ensuring a
2089
2090
2091
    minimum likelihood threshold during sampling.
    """

2092
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2093
2094
    """The seed to use for sampling."""

2095
    frequency_penalty: float | None = 0.0
2096
2097
    """The frequency penalty to use for sampling."""

2098
    repetition_penalty: float | None = None
2099
2100
    """The repetition penalty to use for sampling."""

2101
    presence_penalty: float | None = 0.0
2102
    """The presence penalty to use for sampling."""
2103
2104
2105

    max_completion_tokens: int | None = None
    """The maximum number of tokens to generate."""
2106
    # --8<-- [end:transcription-sampling-params]
2107

2108
2109
    # Default sampling parameters for transcription requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
2110
2111
2112
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
2113
        "top_k": 0,
2114
        "min_p": 0.0,
2115
2116
2117
    }

    def to_sampling_params(
2118
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2119
    ) -> SamplingParams:
2120
2121
2122
2123
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
2124

2125
2126
2127
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2128
2129
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2130
2131
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
2132
2133
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
2134
2135
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
2136
2137
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
2138
2139
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
2140
2141
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
2142
2143
2144
2145

        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )

        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=repetition_penalty,
            presence_penalty=self.presence_penalty,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
            extra_args=self.vllm_xargs,
2163
            skip_clone=True,  # Created fresh per request, safe to skip clone
2164
        )
2165
2166
2167

    @model_validator(mode="before")
    @classmethod
2168
2169
2170
2171
2172
2173
2174
    def validate_transcription_request(cls, data):
        if isinstance(data.get("file"), str):
            raise HTTPException(
                status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
                detail="Expected 'file' to be a file-like object, not 'str'.",
            )

2175
2176
2177
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
2178
2179
2180
2181
2182
2183
2184
2185
2186
            # Find which specific stream option was set
            invalid_param = next(
                (so for so in stream_opts if data.get(so, False)),
                "stream_include_usage",
            )
            raise VLLMValidationError(
                "Stream options can only be defined when `stream=True`.",
                parameter=invalid_param,
            )
2187
2188

        return data
2189
2190
2191


# Transcription response objects
2192
2193
2194
2195
2196
class TranscriptionUsageAudio(OpenAIBaseModel):
    type: Literal["duration"] = "duration"
    seconds: int


2197
2198
2199
class TranscriptionResponse(OpenAIBaseModel):
    text: str
    """The transcribed text."""
2200
    usage: TranscriptionUsageAudio
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217


class TranscriptionWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranscriptionSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

2218
    avg_logprob: float | None = None
2219
2220
2221
2222
2223
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

2224
    compression_ratio: float | None = None
2225
2226
2227
2228
2229
2230
2231
2232
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

2233
    no_speech_prob: float | None = None
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

2252
    tokens: list[int]
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
    """Array of token IDs for the text content."""


class TranscriptionResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The transcribed text."""

2266
    segments: list[TranscriptionSegment] | None = None
2267
2268
    """Segments of the transcribed text and their corresponding details."""

2269
    words: list[TranscriptionWord] | None = None
2270
    """Extracted words and their corresponding timestamps."""
2271
2272


2273
2274
2275
2276
2277
TranscriptionResponseVariant: TypeAlias = (
    TranscriptionResponse | TranscriptionResponseVerbose
)


2278
2279
class TranslationResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
2280
2281
    finish_reason: str | None = None
    stop_reason: int | str | None = None
2282
2283
2284
2285
2286
2287
2288
2289


class TranslationStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
    object: Literal["translation.chunk"] = "translation.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranslationResponseStreamChoice]
2290
    usage: UsageInfo | None = Field(default=None)
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302


class TranslationRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/audio/createTranslation

    file: UploadFile
    """
    The audio file object (not file name) to translate, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

2303
    model: str | None = None
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
    """ID of the model to use.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    # TODO support additional sampling parameters
    # --8<-- [start:translation-sampling-params]
2323
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
2324
2325
    """The seed to use for sampling."""

2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """
    # --8<-- [end:translation-sampling-params]

    # --8<-- [start:translation-extra-params]
2337
    language: str | None = None
2338
2339
2340
2341
2342
2343
2344
    """The language of the input audio we translate from.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy.
    """

2345
    to_language: str | None = None
2346
2347
2348
2349
2350
2351
2352
    """The language of the input audio we translate to.

    Please note that this is not supported by all models, refer to the specific
    model documentation for more details.
    For instance, Whisper only supports `to_language=en`.
    """

2353
    stream: bool | None = False
2354
    """Custom field not present in the original OpenAI definition. When set,
2355
    it will enable output to be streamed in a similar fashion as the Chat
2356
    Completion endpoint.
2357
2358
    """
    # Flattened stream option to simplify form data.
2359
2360
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
2361
2362
2363

    max_completion_tokens: int | None = None
    """The maximum number of tokens to generate."""
2364
2365
2366
2367
2368
2369
2370
2371
    # --8<-- [end:translation-extra-params]

    # Default sampling parameters for translation requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "temperature": 0,
    }

    def to_sampling_params(
2372
        self, default_max_tokens: int, default_sampling_params: dict | None = None
2373
    ) -> SamplingParams:
2374
2375
2376
2377
2378
2379
2380
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
2381
2382
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
2383

2384
2385
2386
2387
2388
2389
2390
        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
2391
            skip_clone=True,  # Created fresh per request, safe to skip clone
2392
        )
2393
2394
2395
2396
2397
2398
2399

    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
2400
2401
2402
2403
2404
2405
2406
2407
2408
            # Find which specific stream option was set
            invalid_param = next(
                (so for so in stream_opts if data.get(so, False)),
                "stream_include_usage",
            )
            raise VLLMValidationError(
                "Stream options can only be defined when `stream=True`.",
                parameter=invalid_param,
            )
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433

        return data


# Translation response objects
class TranslationResponse(OpenAIBaseModel):
    text: str
    """The translated text."""


class TranslationWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranslationSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

2434
    avg_logprob: float | None = None
2435
2436
2437
2438
2439
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

2440
    compression_ratio: float | None = None
2441
2442
2443
2444
2445
2446
2447
2448
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

2449
    no_speech_prob: float | None = None
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

    tokens: list[int]
    """Array of token IDs for the text content."""


class TranslationResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The translated text."""

2482
    segments: list[TranslationSegment] | None = None
2483
2484
    """Segments of the translated text and their corresponding details."""

2485
    words: list[TranslationWord] | None = None
2486
    """Extracted words and their corresponding timestamps."""
2487
2488


2489
2490
2491
TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose


2492
2493
2494
####### Tokens IN <> Tokens OUT #######
class GenerateRequest(BaseModel):
    request_id: str = Field(
2495
        default_factory=random_uuid,
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    token_ids: list[int]
    """The token ids to generate text from."""

    # features: MultiModalFeatureSpec
    # TODO (NickLucche): implement once Renderer work is completed
    features: str | None = None
    """The processed MM inputs for the model."""

    sampling_params: SamplingParams
    """The sampling parameters for the model."""

    model: str | None = None

    stream: bool | None = False
    stream_options: StreamOptions | None = None
    cache_salt: str | None = Field(
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
            "to 256 bit)."
        ),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."
        ),
    )
    kv_transfer_params: dict[str, Any] | None = Field(
        default=None,
        description="KVTransfer parameters used for disaggregated serving.",
    )


class GenerateResponseChoice(BaseModel):
    index: int
    logprobs: ChatCompletionLogProbs | None = None
    # per OpenAI spec this is the default
    finish_reason: str | None = "stop"
    token_ids: list[int] | None = None


class GenerateResponse(BaseModel):
    request_id: str = Field(
2552
        default_factory=random_uuid,
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    choices: list[GenerateResponseChoice]

    prompt_logprobs: list[dict[int, Logprob] | None] | None = None

    kv_transfer_params: dict[str, Any] | None = Field(
        default=None,
        description="KVTransfer parameters used for disaggregated serving.",
    )