protocol.py 42.2 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
6
import json
Zhuohan Li's avatar
Zhuohan Li committed
7
import time
8
from http import HTTPStatus
9
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
Zhuohan Li's avatar
Zhuohan Li committed
10

11
import regex as re
12
import torch
13
from fastapi import HTTPException, UploadFile
14
15
16
17
18
19
from pydantic import (
    BaseModel,
    ConfigDict,
    Field,
    model_validator,
)
Zhuohan Li's avatar
Zhuohan Li committed
20

21
from vllm.entrypoints.chat_utils import make_tool_call_id
22
from vllm.exceptions import VLLMValidationError
23
from vllm.logger import init_logger
24
from vllm.logprobs import Logprob
25
26
27
28
29
30
from vllm.sampling_params import (
    BeamSearchParams,
    RequestOutputKind,
    SamplingParams,
    StructuredOutputsParams,
)
31
32
from vllm.utils import random_uuid
from vllm.utils.import_utils import resolve_obj_by_qualname
33

34
35
logger = init_logger(__name__)

36
_LONG_INFO = torch.iinfo(torch.long)
37

Zhuohan Li's avatar
Zhuohan Li committed
38

39
class OpenAIBaseModel(BaseModel):
40
41
42
    # OpenAI API does allow extra fields
    model_config = ConfigDict(extra="allow")

43
    # Cache class field names
44
    field_names: ClassVar[set[str] | None] = None
45

46
    @model_validator(mode="wrap")
47
    @classmethod
48
49
50
51
    def __log_extra_fields__(cls, data, handler):
        result = handler(data)
        if not isinstance(data, dict):
            return result
52
53
        field_names = cls.field_names
        if field_names is None:
54
55
56
57
            # Get all class field names and their potential aliases
            field_names = set()
            for field_name, field in cls.model_fields.items():
                field_names.add(field_name)
58
                if alias := getattr(field, "alias", None):
59
60
61
62
63
64
                    field_names.add(alias)
            cls.field_names = field_names

        # Compare against both field names and aliases
        if any(k not in field_names for k in data):
            logger.warning(
65
                "The following fields were present in the request but ignored: %s",
66
67
                data.keys() - field_names,
            )
68
        return result
69
70


71
class ErrorInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
72
73
    message: str
    type: str
74
    param: str | None = None
75
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
76
77


78
79
80
81
class ErrorResponse(OpenAIBaseModel):
    error: ErrorInfo


82
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
83
84
85
86
87
88
89
90
91
92
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
93
    group: str | None = None
94
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
95
96


97
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
98
99
100
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
101
    owned_by: str = "vllm"
102
103
104
    root: str | None = None
    parent: str | None = None
    max_model_len: int | None = None
105
    permission: list[ModelPermission] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
106
107


108
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
109
    object: str = "list"
110
    data: list[ModelCard] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
111
112


113
class PromptTokenUsageInfo(OpenAIBaseModel):
114
    cached_tokens: int | None = None
115
116


117
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
118
119
    prompt_tokens: int = 0
    total_tokens: int = 0
120
121
    completion_tokens: int | None = 0
    prompt_tokens_details: PromptTokenUsageInfo | None = None
Zhuohan Li's avatar
Zhuohan Li committed
122
123


124
125
class RequestResponseMetadata(BaseModel):
    request_id: str
126
    final_usage_info: UsageInfo | None = None
127
128


129
130
class JsonSchemaResponseFormat(OpenAIBaseModel):
    name: str
131
    description: str | None = None
132
133
    # schema is the field in openai but that causes conflicts with pydantic so
    # instead use json_schema with an alias
134
135
    json_schema: dict[str, Any] | None = Field(default=None, alias="schema")
    strict: bool | None = None
136
137


138
class LegacyStructuralTag(OpenAIBaseModel):
139
140
141
    begin: str
    # schema is the field, but that causes conflicts with pydantic so
    # instead use structural_tag_schema with an alias
142
    structural_tag_schema: dict[str, Any] | None = Field(default=None, alias="schema")
143
144
145
    end: str


146
class LegacyStructuralTagResponseFormat(OpenAIBaseModel):
147
    type: Literal["structural_tag"]
148
    structures: list[LegacyStructuralTag]
149
150
151
    triggers: list[str]


152
153
154
155
156
157
158
159
160
161
class StructuralTagResponseFormat(OpenAIBaseModel):
    type: Literal["structural_tag"]
    format: Any


AnyStructuralTagResponseFormat: TypeAlias = (
    LegacyStructuralTagResponseFormat | StructuralTagResponseFormat
)


162
class ResponseFormat(OpenAIBaseModel):
163
    # type must be "json_schema", "json_object", or "text"
164
    type: Literal["text", "json_object", "json_schema"]
165
    json_schema: JsonSchemaResponseFormat | None = None
166
167


168
169
170
AnyResponseFormat: TypeAlias = (
    ResponseFormat | StructuralTagResponseFormat | LegacyStructuralTagResponseFormat
)
171
172


173
class StreamOptions(OpenAIBaseModel):
174
175
    include_usage: bool | None = True
    continuous_usage_stats: bool | None = False
176
177


178
179
class FunctionDefinition(OpenAIBaseModel):
    name: str
180
181
    description: str | None = None
    parameters: dict[str, Any] | None = None
182
183


184
185
# extra="forbid" is a workaround to have kwargs as a field,
# see https://github.com/pydantic/pydantic/issues/3125
186
187
class LogitsProcessorConstructor(BaseModel):
    qualname: str
188
189
    args: list[Any] | None = None
    kwargs: dict[str, Any] | None = None
190

191
192
    model_config = ConfigDict(extra="forbid")

193

194
LogitsProcessors = list[str | LogitsProcessorConstructor]
195
196


197
def get_logits_processors(
198
199
    processors: LogitsProcessors | None, pattern: str | None
) -> list[Any] | None:
200
201
202
    if processors and pattern:
        logits_processors = []
        for processor in processors:
203
            qualname = processor if isinstance(processor, str) else processor.qualname
204
205
206
207
            if not re.match(pattern, qualname):
                raise ValueError(
                    f"Logits processor '{qualname}' is not allowed by this "
                    "server. See --logits-processor-pattern engine argument "
208
209
                    "for more information."
                )
210
211
212
213
214
215
216
            try:
                logits_processor = resolve_obj_by_qualname(qualname)
            except Exception as e:
                raise ValueError(
                    f"Logits processor '{qualname}' could not be resolved: {e}"
                ) from e
            if isinstance(processor, LogitsProcessorConstructor):
217
218
219
                logits_processor = logits_processor(
                    *processor.args or [], **processor.kwargs or {}
                )
220
221
222
223
224
            logits_processors.append(logits_processor)
        return logits_processors
    elif processors:
        raise ValueError(
            "The `logits_processors` argument is not supported by this "
225
            "server. See --logits-processor-pattern engine argument "
226
227
            "for more information."
        )
228
229
230
    return None


231
class CompletionRequest(OpenAIBaseModel):
232
233
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
234
235
236
237
238
239
240
    model: str | None = None
    prompt: list[int] | list[list[int]] | str | list[str] | None = None
    echo: bool | None = False
    frequency_penalty: float | None = 0.0
    logit_bias: dict[str, float] | None = None
    logprobs: int | None = None
    max_tokens: int | None = 16
241
    n: int = 1
242
243
244
245
246
247
248
249
250
    presence_penalty: float | None = 0.0
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
    stop: str | list[str] | None = []
    stream: bool | None = False
    stream_options: StreamOptions | None = None
    suffix: str | None = None
    temperature: float | None = None
    top_p: float | None = None
    user: str | None = None
251

252
    # --8<-- [start:completion-sampling-params]
253
    use_beam_search: bool = False
254
255
256
    top_k: int | None = None
    min_p: float | None = None
    repetition_penalty: float | None = None
257
    length_penalty: float = 1.0
258
    stop_token_ids: list[int] | None = []
259
260
261
262
263
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
264
265
266
    truncate_prompt_tokens: Annotated[int, Field(ge=-1, le=_LONG_INFO.max)] | None = (
        None
    )
267
268
    allowed_token_ids: list[int] | None = None
    prompt_logprobs: int | None = None
269
    # --8<-- [end:completion-sampling-params]
270

271
    # --8<-- [start:completion-extra-params]
272
    prompt_embeds: bytes | list[bytes] | None = None
273
274
    add_special_tokens: bool = Field(
        default=True,
275
        description=(
276
            "If true (the default), special tokens (e.g. BOS) will be added to "
277
278
            "the prompt."
        ),
279
    )
280
    response_format: AnyResponseFormat | None = Field(
281
        default=None,
282
283
284
285
286
        description=(
            "Similar to chat completion, this parameter specifies the format "
            "of output. Only {'type': 'json_object'}, {'type': 'json_schema'}"
            ", {'type': 'structural_tag'}, or {'type': 'text' } is supported."
        ),
287
    )
288
    structured_outputs: StructuredOutputsParams | None = Field(
289
        default=None,
290
        description="Additional kwargs for structured outputs",
291
    )
292
293
294
295
296
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
297
298
            "if the served model does not use priority scheduling."
        ),
299
    )
300
    request_id: str = Field(
301
        default_factory=random_uuid,
302
303
304
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
305
306
            "through out the inference process and return in response."
        ),
307
    )
308
    logits_processors: LogitsProcessors | None = Field(
309
310
311
312
313
314
315
316
317
        default=None,
        description=(
            "A list of either qualified names of logits processors, or "
            "constructor objects, to apply when sampling. A constructor is "
            "a JSON object with a required 'qualname' field specifying the "
            "qualified name of the processor class/factory, and optional "
            "'args' and 'kwargs' fields containing positional and keyword "
            "arguments. For example: {'qualname': "
            "'my_module.MyLogitsProcessor', 'args': [1, 2], 'kwargs': "
318
319
320
            "{'param': 'value'}}."
        ),
    )
321

322
    return_tokens_as_token_ids: bool | None = Field(
323
324
325
326
        default=None,
        description=(
            "If specified with 'logprobs', tokens are represented "
            " as strings of the form 'token_id:{token_id}' so that tokens "
327
328
329
            "that are not JSON-encodable can be identified."
        ),
    )
330
    return_token_ids: bool | None = Field(
331
332
333
334
335
336
        default=None,
        description=(
            "If specified, the result will include token IDs alongside the "
            "generated text. In streaming mode, prompt_token_ids is included "
            "only in the first chunk, and token_ids contains the delta tokens "
            "for each chunk. This is useful for debugging or when you "
337
338
339
            "need to map generated text back to input tokens."
        ),
    )
340

341
    cache_salt: str | None = Field(
342
343
344
345
346
347
348
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
349
            "to 256 bit)."
350
351
        ),
    )
352

353
    kv_transfer_params: dict[str, Any] | None = Field(
Robert Shaw's avatar
Robert Shaw committed
354
        default=None,
355
356
        description="KVTransfer parameters used for disaggregated serving.",
    )
Robert Shaw's avatar
Robert Shaw committed
357

358
    vllm_xargs: dict[str, str | int | float] | None = Field(
359
        default=None,
360
361
362
363
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
364
365
    )

366
    # --8<-- [end:completion-extra-params]
Zhuohan Li's avatar
Zhuohan Li committed
367

368
369
370
371
372
    # Default sampling parameters for completion requests
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
373
        "top_k": 0,
374
375
376
377
        "min_p": 0.0,
    }

    def to_beam_search_params(
378
379
        self,
        max_tokens: int,
380
        default_sampling_params: dict | None = None,
381
382
383
    ) -> BeamSearchParams:
        if default_sampling_params is None:
            default_sampling_params = {}
384
        n = self.n if self.n is not None else 1
385
386
387

        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get("temperature", 1.0)
388
389
390
391
392
393

        return BeamSearchParams(
            beam_width=n,
            max_tokens=max_tokens,
            ignore_eos=self.ignore_eos,
            temperature=temperature,
394
            length_penalty=self.length_penalty,
395
396
            include_stop_str_in_output=self.include_stop_str_in_output,
        )
397

398
    def to_sampling_params(
399
        self,
400
        max_tokens: int,
401
402
        logits_processor_pattern: str | None,
        default_sampling_params: dict | None = None,
403
    ) -> SamplingParams:
404
405
        if default_sampling_params is None:
            default_sampling_params = {}
406

407
408
409
410
411
412
413
414
        # Default parameters
        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
415
416
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
417
418
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
419
420
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
421
422
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
423
424
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
425
426
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
427
428
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
429

430
431
432
433
        prompt_logprobs = self.prompt_logprobs
        if prompt_logprobs is None and self.echo:
            prompt_logprobs = self.logprobs

434
435
        echo_without_generation = self.echo and self.max_tokens == 0

436
437
438
439
440
441
442
443
444
445
446
447
        response_format = self.response_format
        if response_format is not None:
            # If structured outputs wasn't already enabled,
            # we must enable it for these features to work
            if self.structured_outputs is None:
                self.structured_outputs = StructuredOutputsParams()

            # Set structured output params for response format
            if response_format.type == "json_object":
                self.structured_outputs.json_object = True
            elif response_format.type == "json_schema":
                json_schema = response_format.json_schema
448
                assert json_schema is not None
449
450
451
                self.structured_outputs.json = json_schema.json_schema
            elif response_format.type == "structural_tag":
                structural_tag = response_format
452
                assert structural_tag is not None and isinstance(
453
454
455
456
457
                    structural_tag,
                    (
                        LegacyStructuralTagResponseFormat,
                        StructuralTagResponseFormat,
                    ),
458
459
                )
                s_tag_obj = structural_tag.model_dump(by_alias=True)
460
                self.structured_outputs.structural_tag = json.dumps(s_tag_obj)
461

462
463
464
465
        extra_args: dict[str, Any] = self.vllm_xargs if self.vllm_xargs else {}
        if self.kv_transfer_params:
            # Pass in kv_transfer_params via extra_args
            extra_args["kv_transfer_params"] = self.kv_transfer_params
466
        return SamplingParams.from_optional(
467
468
469
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
470
471
472
473
474
            repetition_penalty=repetition_penalty,
            temperature=temperature,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
Nick Hill's avatar
Nick Hill committed
475
            seed=self.seed,
476
477
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
478
            logprobs=self.logprobs,
479
            ignore_eos=self.ignore_eos,
480
            max_tokens=max_tokens if not echo_without_generation else 1,
481
            min_tokens=self.min_tokens,
482
            prompt_logprobs=prompt_logprobs,
483
            skip_special_tokens=self.skip_special_tokens,
484
            spaces_between_special_tokens=self.spaces_between_special_tokens,
485
            include_stop_str_in_output=self.include_stop_str_in_output,
486
487
488
            logits_processors=get_logits_processors(
                self.logits_processors, logits_processor_pattern
            ),
489
            truncate_prompt_tokens=self.truncate_prompt_tokens,
490
491
492
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
493
            structured_outputs=self.structured_outputs,
494
            logit_bias=self.logit_bias,
Robert Shaw's avatar
Robert Shaw committed
495
            allowed_token_ids=self.allowed_token_ids,
496
            extra_args=extra_args or None,
497
            skip_clone=True,  # Created fresh per request, safe to skip clone
498
        )
499

500
501
    @model_validator(mode="before")
    @classmethod
502
    def check_structured_outputs_count(cls, data):
503
        if data.get("structured_outputs", None) is None:
504
505
            return data

506
        structured_outputs_kwargs = data["structured_outputs"]
507
508
        count = sum(
            structured_outputs_kwargs.get(k) is not None
509
510
            for k in ("json", "regex", "choice")
        )
511
        if count > 1:
512
            raise VLLMValidationError(
513
                "You can only use one kind of constraints for structured "
514
515
                "outputs ('json', 'regex' or 'choice').",
                parameter="structured_outputs",
516
            )
517
518
        return data

519
520
521
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
522
        if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
523
            if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
524
525
526
                raise VLLMValidationError(
                    "`prompt_logprobs` are not available when `stream=True`.",
                    parameter="prompt_logprobs",
527
                )
528

529
            if prompt_logprobs < 0 and prompt_logprobs != -1:
530
531
532
533
534
                raise VLLMValidationError(
                    "`prompt_logprobs` must be a positive value or -1.",
                    parameter="prompt_logprobs",
                    value=prompt_logprobs,
                )
535
        if (logprobs := data.get("logprobs")) is not None and logprobs < 0:
536
537
538
539
540
            raise VLLMValidationError(
                "`logprobs` must be a positive value.",
                parameter="logprobs",
                value=logprobs,
            )
541

542
543
        return data

544
545
546
547
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
548
549
550
551
            raise VLLMValidationError(
                "Stream options can only be defined when `stream=True`.",
                parameter="stream_options",
            )
552

553
554
        return data

555
556
557
    @model_validator(mode="before")
    @classmethod
    def validate_prompt_and_prompt_embeds(cls, data):
558
559
560
        prompt = data.get("prompt")
        prompt_embeds = data.get("prompt_embeds")

561
562
563
564
        prompt_is_empty = prompt is None or (isinstance(prompt, str) and prompt == "")
        embeds_is_empty = prompt_embeds is None or (
            isinstance(prompt_embeds, list) and len(prompt_embeds) == 0
        )
565
566

        if prompt_is_empty and embeds_is_empty:
567
            raise ValueError(
568
569
570
                "Either prompt or prompt_embeds must be provided and non-empty."
            )

571
572
        return data

573
574
575
    @model_validator(mode="before")
    @classmethod
    def check_cache_salt_support(cls, data):
576
577
578
579
580
581
        if data.get("cache_salt") is not None and (
            not isinstance(data["cache_salt"], str) or not data["cache_salt"]
        ):
            raise ValueError(
                "Parameter 'cache_salt' must be a non-empty string if provided."
            )
582
583
        return data

Zhuohan Li's avatar
Zhuohan Li committed
584

585
class CompletionLogProbs(OpenAIBaseModel):
586
    text_offset: list[int] = Field(default_factory=list)
587
    token_logprobs: list[float | None] = Field(default_factory=list)
588
    tokens: list[str] = Field(default_factory=list)
589
    top_logprobs: list[dict[str, float] | None] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
590
591


592
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
593
594
    index: int
    text: str
595
596
597
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
598
599
600
601
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
602
603
            "including encountering the EOS token"
        ),
604
    )
605
606
607
    token_ids: list[int] | None = None  # For response
    prompt_logprobs: list[dict[int, Logprob] | None] | None = None
    prompt_token_ids: list[int] | None = None  # For prompt
Zhuohan Li's avatar
Zhuohan Li committed
608
609


610
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
611
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
612
    object: Literal["text_completion"] = "text_completion"
Zhuohan Li's avatar
Zhuohan Li committed
613
614
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
615
    choices: list[CompletionResponseChoice]
616
617
    service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
    system_fingerprint: str | None = None
Zhuohan Li's avatar
Zhuohan Li committed
618
    usage: UsageInfo
619
620

    # vLLM-specific fields that are not in OpenAI spec
621
    kv_transfer_params: dict[str, Any] | None = Field(
622
623
        default=None, description="KVTransfer parameters."
    )
Zhuohan Li's avatar
Zhuohan Li committed
624
625


626
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
627
628
    index: int
    text: str
629
630
631
    logprobs: CompletionLogProbs | None = None
    finish_reason: str | None = None
    stop_reason: int | str | None = Field(
632
633
634
635
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
636
637
            "including encountering the EOS token"
        ),
638
    )
639
640
    # not part of the OpenAI spec but for tracing the tokens
    # prompt tokens is put into choice to align with CompletionResponseChoice
641
642
    prompt_token_ids: list[int] | None = None
    token_ids: list[int] | None = None
Zhuohan Li's avatar
Zhuohan Li committed
643
644


645
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
646
647
648
649
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
650
    choices: list[CompletionResponseStreamChoice]
651
    usage: UsageInfo | None = Field(default=None)
652
653


654
655
656
657
658
659
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
660
    id: str = Field(default_factory=make_tool_call_id)
661
662
663
664
    type: Literal["function"] = "function"
    function: FunctionCall


665
class DeltaFunctionCall(BaseModel):
666
667
    name: str | None = None
    arguments: str | None = None
668
669
670
671


# a tool call delta where everything is optional
class DeltaToolCall(OpenAIBaseModel):
672
673
    id: str | None = None
    type: Literal["function"] | None = None
674
    index: int
675
    function: DeltaFunctionCall | None = None
676
677
678
679
680
681
682


class ExtractedToolCallInformation(BaseModel):
    # indicate if tools were called
    tools_called: bool

    # extracted tool calls
683
    tool_calls: list[ToolCall]
684
685
686

    # content - per OpenAI spec, content AND tool calls can be returned rarely
    # But some models will do this intentionally
687
    content: str | None = None
688
689


690
class DeltaMessage(OpenAIBaseModel):
691
692
    role: str | None = None
    content: str | None = None
693
    reasoning: str | None = None
694
    reasoning_content: str | None = None
695
    """Deprecated: use `reasoning` instead."""
696
    tool_calls: list[DeltaToolCall] = Field(default_factory=list)
697

698
699
700
701
702
703
    @model_validator(mode="after")
    def handle_deprecated_reasoning_content(self):
        """Copy reasoning to reasoning_content for backward compatibility."""
        self.reasoning_content = self.reasoning
        return self

704

705
706
class TranscriptionResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
707
708
    finish_reason: str | None = None
    stop_reason: int | str | None = None
709
710
711
712
713
714
715
716


class TranscriptionStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsc-{random_uuid()}")
    object: Literal["transcription.chunk"] = "transcription.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranscriptionResponseStreamChoice]
717
    usage: UsageInfo | None = Field(default=None)
718
719


720
## Protocols for Audio
721
AudioResponseFormat: TypeAlias = Literal["json", "text", "srt", "verbose_json", "vtt"]
722
723
724
725


class TranscriptionRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
726
    # https://platform.openai.com/docs/api-reference/audio/createTranscription
727
728
729
730
731
732
733

    file: UploadFile
    """
    The audio file object (not file name) to transcribe, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

734
    model: str | None = None
735
736
737
    """ID of the model to use.
    """

738
    language: str | None = None
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
    """The language of the input audio.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy and latency.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    ## TODO (varun) : Support if set to 0, certain thresholds are met !!

762
    timestamp_granularities: list[Literal["word", "segment"]] = Field(
763
764
        alias="timestamp_granularities[]", default=[]
    )
765
766
767
768
769
770
771
772
    """The timestamp granularities to populate for this transcription.

    `response_format` must be set `verbose_json` to use timestamp granularities.
    Either or both of these options are supported: `word`, or `segment`. Note:
    There is no additional latency for segment timestamps, but generating word
    timestamps incurs additional latency.
    """

773
    stream: bool | None = False
774
    """When set, it will enable output to be streamed in a similar fashion
775
    as the Chat Completion endpoint.
776
    """
777
    # --8<-- [start:transcription-extra-params]
778
    # Flattened stream option to simplify form data.
779
780
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
781

782
    vllm_xargs: dict[str, str | int | float] | None = Field(
783
        default=None,
784
785
786
787
        description=(
            "Additional request parameters with string or "
            "numeric values, used by custom extensions."
        ),
788
    )
789
    # --8<-- [end:transcription-extra-params]
790

791
    to_language: str | None = None
792
793
    """The language of the output audio we transcribe to.

794
    Please note that this is not currently used by supported models at this
795
796
797
    time, but it is a placeholder for future use, matching translation api.
    """

798
    # --8<-- [start:transcription-sampling-params]
799
800
801
802
803
804
805
806
807
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """

808
    top_p: float | None = None
809
    """Enables nucleus (top-p) sampling, where tokens are selected from the
810
811
812
    smallest possible set whose cumulative probability exceeds `p`.
    """

813
    top_k: int | None = None
814
815
    """Limits sampling to the `k` most probable tokens at each step."""

816
    min_p: float | None = None
817
    """Filters out tokens with a probability lower than `min_p`, ensuring a
818
819
820
    minimum likelihood threshold during sampling.
    """

821
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
822
823
    """The seed to use for sampling."""

824
    frequency_penalty: float | None = 0.0
825
826
    """The frequency penalty to use for sampling."""

827
    repetition_penalty: float | None = None
828
829
    """The repetition penalty to use for sampling."""

830
    presence_penalty: float | None = 0.0
831
    """The presence penalty to use for sampling."""
832
833
834

    max_completion_tokens: int | None = None
    """The maximum number of tokens to generate."""
835
    # --8<-- [end:transcription-sampling-params]
836

837
838
    # Default sampling parameters for transcription requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
839
840
841
        "repetition_penalty": 1.0,
        "temperature": 1.0,
        "top_p": 1.0,
842
        "top_k": 0,
843
        "min_p": 0.0,
844
845
846
    }

    def to_sampling_params(
847
        self, default_max_tokens: int, default_sampling_params: dict | None = None
848
    ) -> SamplingParams:
849
850
851
852
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
853

854
855
856
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
857
858
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
859
860
        if (top_p := self.top_p) is None:
            top_p = default_sampling_params.get(
861
862
                "top_p", self._DEFAULT_SAMPLING_PARAMS["top_p"]
            )
863
864
        if (top_k := self.top_k) is None:
            top_k = default_sampling_params.get(
865
866
                "top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
            )
867
868
        if (min_p := self.min_p) is None:
            min_p = default_sampling_params.get(
869
870
                "min_p", self._DEFAULT_SAMPLING_PARAMS["min_p"]
            )
871
872
873
874

        if (repetition_penalty := self.repetition_penalty) is None:
            repetition_penalty = default_sampling_params.get(
                "repetition_penalty",
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
                self._DEFAULT_SAMPLING_PARAMS["repetition_penalty"],
            )

        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            top_p=top_p,
            top_k=top_k,
            min_p=min_p,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=repetition_penalty,
            presence_penalty=self.presence_penalty,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
            extra_args=self.vllm_xargs,
892
            skip_clone=True,  # Created fresh per request, safe to skip clone
893
        )
894
895
896

    @model_validator(mode="before")
    @classmethod
897
898
899
900
901
902
903
    def validate_transcription_request(cls, data):
        if isinstance(data.get("file"), str):
            raise HTTPException(
                status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
                detail="Expected 'file' to be a file-like object, not 'str'.",
            )

904
905
906
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
907
908
909
910
911
912
913
914
915
            # Find which specific stream option was set
            invalid_param = next(
                (so for so in stream_opts if data.get(so, False)),
                "stream_include_usage",
            )
            raise VLLMValidationError(
                "Stream options can only be defined when `stream=True`.",
                parameter=invalid_param,
            )
916
917

        return data
918
919
920


# Transcription response objects
921
922
923
924
925
class TranscriptionUsageAudio(OpenAIBaseModel):
    type: Literal["duration"] = "duration"
    seconds: int


926
927
928
class TranscriptionResponse(OpenAIBaseModel):
    text: str
    """The transcribed text."""
929
    usage: TranscriptionUsageAudio
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946


class TranscriptionWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranscriptionSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

947
    avg_logprob: float | None = None
948
949
950
951
952
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

953
    compression_ratio: float | None = None
954
955
956
957
958
959
960
961
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

962
    no_speech_prob: float | None = None
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

981
    tokens: list[int]
982
983
984
985
986
987
988
989
990
991
992
993
994
    """Array of token IDs for the text content."""


class TranscriptionResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The transcribed text."""

995
    segments: list[TranscriptionSegment] | None = None
996
997
    """Segments of the transcribed text and their corresponding details."""

998
    words: list[TranscriptionWord] | None = None
999
    """Extracted words and their corresponding timestamps."""
1000
1001


1002
1003
1004
1005
1006
TranscriptionResponseVariant: TypeAlias = (
    TranscriptionResponse | TranscriptionResponseVerbose
)


1007
1008
class TranslationResponseStreamChoice(OpenAIBaseModel):
    delta: DeltaMessage
1009
1010
    finish_reason: str | None = None
    stop_reason: int | str | None = None
1011
1012
1013
1014
1015
1016
1017
1018


class TranslationStreamResponse(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"trsl-{random_uuid()}")
    object: Literal["translation.chunk"] = "translation.chunk"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: list[TranslationResponseStreamChoice]
1019
    usage: UsageInfo | None = Field(default=None)
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031


class TranslationRequest(OpenAIBaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/audio/createTranslation

    file: UploadFile
    """
    The audio file object (not file name) to translate, in one of these
    formats: flac, mp3, mp4, mpeg, mpga, m4a, ogg, wav, or webm.
    """

1032
    model: str | None = None
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
    """ID of the model to use.
    """

    prompt: str = Field(default="")
    """An optional text to guide the model's style or continue a previous audio
    segment.

    The [prompt](https://platform.openai.com/docs/guides/speech-to-text#prompting)
    should match the audio language.
    """

    response_format: AudioResponseFormat = Field(default="json")
    """
    The format of the output, in one of these options: `json`, `text`, `srt`,
    `verbose_json`, or `vtt`.
    """

    # TODO support additional sampling parameters
    # --8<-- [start:translation-sampling-params]
1052
    seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
1053
1054
    """The seed to use for sampling."""

1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
    temperature: float = Field(default=0.0)
    """The sampling temperature, between 0 and 1.

    Higher values like 0.8 will make the output more random, while lower values
    like 0.2 will make it more focused / deterministic. If set to 0, the model
    will use [log probability](https://en.wikipedia.org/wiki/Log_probability)
    to automatically increase the temperature until certain thresholds are hit.
    """
    # --8<-- [end:translation-sampling-params]

    # --8<-- [start:translation-extra-params]
1066
    language: str | None = None
1067
1068
1069
1070
1071
1072
1073
    """The language of the input audio we translate from.

    Supplying the input language in
    [ISO-639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) format
    will improve accuracy.
    """

1074
    to_language: str | None = None
1075
1076
1077
1078
1079
1080
1081
    """The language of the input audio we translate to.

    Please note that this is not supported by all models, refer to the specific
    model documentation for more details.
    For instance, Whisper only supports `to_language=en`.
    """

1082
    stream: bool | None = False
1083
    """Custom field not present in the original OpenAI definition. When set,
1084
    it will enable output to be streamed in a similar fashion as the Chat
1085
    Completion endpoint.
1086
1087
    """
    # Flattened stream option to simplify form data.
1088
1089
    stream_include_usage: bool | None = False
    stream_continuous_usage_stats: bool | None = False
1090
1091
1092

    max_completion_tokens: int | None = None
    """The maximum number of tokens to generate."""
1093
1094
1095
1096
1097
1098
1099
1100
    # --8<-- [end:translation-extra-params]

    # Default sampling parameters for translation requests.
    _DEFAULT_SAMPLING_PARAMS: dict = {
        "temperature": 0,
    }

    def to_sampling_params(
1101
        self, default_max_tokens: int, default_sampling_params: dict | None = None
1102
    ) -> SamplingParams:
1103
1104
1105
1106
1107
1108
1109
        max_tokens = default_max_tokens

        if default_sampling_params is None:
            default_sampling_params = {}
        # Default parameters
        if (temperature := self.temperature) is None:
            temperature = default_sampling_params.get(
1110
1111
                "temperature", self._DEFAULT_SAMPLING_PARAMS["temperature"]
            )
1112

1113
1114
1115
1116
1117
1118
1119
        return SamplingParams.from_optional(
            temperature=temperature,
            max_tokens=max_tokens,
            seed=self.seed,
            output_kind=RequestOutputKind.DELTA
            if self.stream
            else RequestOutputKind.FINAL_ONLY,
1120
            skip_clone=True,  # Created fresh per request, safe to skip clone
1121
        )
1122
1123
1124
1125
1126
1127
1128

    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        stream_opts = ["stream_include_usage", "stream_continuous_usage_stats"]
        stream = data.get("stream", False)
        if any(bool(data.get(so, False)) for so in stream_opts) and not stream:
1129
1130
1131
1132
1133
1134
1135
1136
1137
            # Find which specific stream option was set
            invalid_param = next(
                (so for so in stream_opts if data.get(so, False)),
                "stream_include_usage",
            )
            raise VLLMValidationError(
                "Stream options can only be defined when `stream=True`.",
                parameter=invalid_param,
            )
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162

        return data


# Translation response objects
class TranslationResponse(OpenAIBaseModel):
    text: str
    """The translated text."""


class TranslationWord(OpenAIBaseModel):
    end: float
    """End time of the word in seconds."""

    start: float
    """Start time of the word in seconds."""

    word: str
    """The text content of the word."""


class TranslationSegment(OpenAIBaseModel):
    id: int
    """Unique identifier of the segment."""

1163
    avg_logprob: float | None = None
1164
1165
1166
1167
1168
    """Average logprob of the segment.

    If the value is lower than -1, consider the logprobs failed.
    """

1169
    compression_ratio: float | None = None
1170
1171
1172
1173
1174
1175
1176
1177
    """Compression ratio of the segment.

    If the value is greater than 2.4, consider the compression failed.
    """

    end: float
    """End time of the segment in seconds."""

1178
    no_speech_prob: float | None = None
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
    """Probability of no speech in the segment.

    If the value is higher than 1.0 and the `avg_logprob` is below -1, consider
    this segment silent.
    """

    seek: int
    """Seek offset of the segment."""

    start: float
    """Start time of the segment in seconds."""

    temperature: float
    """Temperature parameter used for generating the segment."""

    text: str
    """Text content of the segment."""

    tokens: list[int]
    """Array of token IDs for the text content."""


class TranslationResponseVerbose(OpenAIBaseModel):
    duration: str
    """The duration of the input audio."""

    language: str
    """The language of the input audio."""

    text: str
    """The translated text."""

1211
    segments: list[TranslationSegment] | None = None
1212
1213
    """Segments of the translated text and their corresponding details."""

1214
    words: list[TranslationWord] | None = None
1215
    """Extracted words and their corresponding timestamps."""
1216
1217


1218
1219
1220
TranslationResponseVariant: TypeAlias = TranslationResponse | TranslationResponseVerbose


1221
1222
1223
####### Tokens IN <> Tokens OUT #######
class GenerateRequest(BaseModel):
    request_id: str = Field(
1224
        default_factory=random_uuid,
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
        description=(
            "The request_id related to this request. If the caller does "
            "not set it, a random_uuid will be generated. This id is used "
            "through out the inference process and return in response."
        ),
    )
    token_ids: list[int]
    """The token ids to generate text from."""

    # features: MultiModalFeatureSpec
    # TODO (NickLucche): implement once Renderer work is completed
    features: str | None = None
    """The processed MM inputs for the model."""

    sampling_params: SamplingParams
    """The sampling parameters for the model."""

    model: str | None = None

    stream: bool | None = False
    stream_options: StreamOptions | None = None
    cache_salt: str | None = Field(
        default=None,
        description=(
            "If specified, the prefix cache will be salted with the provided "
            "string to prevent an attacker to guess prompts in multi-user "
            "environments. The salt should be random, protected from "
            "access by 3rd parties, and long enough to be "
            "unpredictable (e.g., 43 characters base64-encoded, corresponding "
            "to 256 bit)."
        ),
    )
    priority: int = Field(
        default=0,
        description=(
            "The priority of the request (lower means earlier handling; "
            "default: 0). Any priority other than 0 will raise an error "
            "if the served model does not use priority scheduling."
        ),
    )
    kv_transfer_params: dict[str, Any] | None = Field(
        default=None,
        description="KVTransfer parameters used for disaggregated serving.",
    )