protocol.py 25.3 KB
Newer Older
1
2
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
Zhuohan Li's avatar
Zhuohan Li committed
3
import time
4
from argparse import Namespace
5
from typing import Any, Dict, List, Literal, Optional, Union
Zhuohan Li's avatar
Zhuohan Li committed
6

7
import torch
8
from pydantic import BaseModel, ConfigDict, Field, model_validator
9
from transformers import PreTrainedTokenizer
10
from typing_extensions import Annotated
Zhuohan Li's avatar
Zhuohan Li committed
11

12
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
13
from vllm.entrypoints.openai.logits_processors import get_logits_processors
14
from vllm.pooling_params import PoolingParams
15
from vllm.sampling_params import LogitsProcessor, SamplingParams
16
from vllm.utils import random_uuid
17

18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
# torch is mocked during docs generation,
# so we have to provide the values as literals
_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)

try:
    from sphinx.ext.autodoc.mock import _MockModule

    if isinstance(torch, _MockModule):
        _LONG_INFO = _MOCK_LONG_INFO
    else:
        _LONG_INFO = torch.iinfo(torch.long)
except ModuleNotFoundError:
    _LONG_INFO = torch.iinfo(torch.long)

assert _LONG_INFO.min == _MOCK_LONG_INFO.min
assert _LONG_INFO.max == _MOCK_LONG_INFO.max

Zhuohan Li's avatar
Zhuohan Li committed
35

36
37
38
39
40
41
class OpenAIBaseModel(BaseModel):
    # OpenAI API does not allow extra fields
    model_config = ConfigDict(extra="forbid")


class ErrorResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
42
43
44
45
    object: str = "error"
    message: str
    type: str
    param: Optional[str] = None
46
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
47
48


49
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
50
51
52
53
54
55
56
57
58
59
60
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
    group: Optional[str] = None
61
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
62
63


64
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
65
66
67
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
68
    owned_by: str = "vllm"
Zhuohan Li's avatar
Zhuohan Li committed
69
70
    root: Optional[str] = None
    parent: Optional[str] = None
71
    max_model_len: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
72
73
74
    permission: List[ModelPermission] = Field(default_factory=list)


75
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
76
77
78
79
    object: str = "list"
    data: List[ModelCard] = Field(default_factory=list)


80
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
81
82
83
84
85
    prompt_tokens: int = 0
    total_tokens: int = 0
    completion_tokens: Optional[int] = 0


86
class ResponseFormat(OpenAIBaseModel):
87
    # type must be "json_object" or "text"
88
    type: Literal["text", "json_object"]
89
90


91
class StreamOptions(OpenAIBaseModel):
92
93
    include_usage: Optional[bool] = True
    continuous_usage_stats: Optional[bool] = True
94
95


96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
class FunctionDefinition(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    parameters: Optional[Dict[str, Any]] = None


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


116
class ChatCompletionRequest(OpenAIBaseModel):
117
118
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
119
    messages: List[ChatCompletionMessageParam]
120
121
122
123
    model: str
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
    logprobs: Optional[bool] = False
124
    top_logprobs: Optional[int] = 0
125
    max_tokens: Optional[int] = None
126
127
128
    n: Optional[int] = 1
    presence_penalty: Optional[float] = 0.0
    response_format: Optional[ResponseFormat] = None
129
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
130
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
131
    stream: Optional[bool] = False
132
    stream_options: Optional[StreamOptions] = None
133
134
    temperature: Optional[float] = 0.7
    top_p: Optional[float] = 1.0
135
136
137
    tools: Optional[List[ChatCompletionToolsParam]] = None
    tool_choice: Optional[Union[Literal["none"],
                                ChatCompletionNamedToolChoiceParam]] = "none"
Zhuohan Li's avatar
Zhuohan Li committed
138
    user: Optional[str] = None
139
140

    # doc: begin-chat-completion-sampling-params
141
    best_of: Optional[int] = None
142
143
144
145
146
147
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
    early_stopping: bool = False
148
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
149
150
151
152
153
154
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
155
156
157
    # doc: end-chat-completion-sampling-params

    # doc: begin-chat-completion-extra-params
158
    echo: bool = Field(
159
160
161
162
163
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
            "if they belong to the same role."),
    )
164
    add_generation_prompt: bool = Field(
165
166
167
168
169
170
        default=True,
        description=
        ("If true, the generation prompt will be added to the chat template. "
         "This is a parameter used by chat template in tokenizer config of the "
         "model."),
    )
171
    add_special_tokens: bool = Field(
172
173
174
175
176
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
177
            "special tokens so this should be set to false (as is the "
178
179
            "default)."),
    )
180
181
182
183
184
185
186
187
188
189
190
191
192
    documents: Optional[List[Dict[str, str]]] = Field(
        default=None,
        description=
        ("A list of dicts representing documents that will be accessible to "
         "the model if it is performing RAG (retrieval-augmented generation)."
         " If the template does not support RAG, this argument will have no "
         "effect. We recommend that each document should be a dict containing "
         "\"title\" and \"text\" keys."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
193
194
195
            "As of transformers v4.44, default chat template is no longer "
            "allowed, so you must provide a chat template if the tokenizer "
            "does not define one."),
196
197
198
199
200
201
    )
    chat_template_kwargs: Optional[Dict[str, Any]] = Field(
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
        description=("If specified, the output will follow the JSON schema."),
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
221
222
223
224
225
226
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be either "
            "'outlines' / 'lm-format-enforcer'"))
227
228
229
230
231
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
232
233

    # doc: end-chat-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
234

235
236
237
238
239
240
241
    def to_sampling_params(
            self, tokenizer: PreTrainedTokenizer,
            guided_decode_logits_processor: Optional[LogitsProcessor],
            default_max_tokens: int) -> SamplingParams:
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens
242

243
        # We now allow logprobs being true without top_logrobs.
244
245
246
247
248
        logits_processors = get_logits_processors(
            logit_bias=self.logit_bias,
            allowed_token_ids=None,
            tokenizer=tokenizer,
        )
249
250
        if guided_decode_logits_processor:
            logits_processors.append(guided_decode_logits_processor)
251

252
253
        return SamplingParams(
            n=self.n,
254
            best_of=self.best_of,
255
256
257
258
259
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
260
            top_k=self.top_k,
261
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
262
            seed=self.seed,
263
264
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
265
266
            logprobs=self.top_logprobs if self.logprobs else None,
            prompt_logprobs=self.top_logprobs if self.echo else None,
267
            ignore_eos=self.ignore_eos,
268
            max_tokens=max_tokens,
269
            min_tokens=self.min_tokens,
270
            use_beam_search=self.use_beam_search,
271
            early_stopping=self.early_stopping,
272
273
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
274
275
            include_stop_str_in_output=self.include_stop_str_in_output,
            length_penalty=self.length_penalty,
276
            logits_processors=logits_processors,
277
            truncate_prompt_tokens=self.truncate_prompt_tokens,
278
279
        )

280
281
282
283
284
285
286
287
288
    @model_validator(mode='before')
    @classmethod
    def validate_stream_options(cls, values):
        if (values.get('stream_options') is not None
                and not values.get('stream')):
            raise ValueError(
                "stream_options can only be set if stream is true")
        return values

289
290
291
292
293
294
295
296
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
297
        # you can only use one kind of guided decoding
298
299
300
301
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
        # you can only either use guided decoding or tools, not both
        if guide_count > 1 and "tool_choice" in data and data[
                "tool_choice"] != "none":
            raise ValueError(
                "You can only either use guided decoding or tools, not both.")
        return data

    @model_validator(mode="before")
    @classmethod
    def check_tool_choice(cls, data):
        if "tool_choice" in data and data["tool_choice"] != "none":
            if not isinstance(data["tool_choice"], dict):
                raise ValueError("Currently only named tools are supported.")
            if "tools" not in data or data["tools"] is None:
                raise ValueError(
                    "When using `tool_choice`, `tools` must be set.")
318
319
        return data

320
321
322
323
324
325
326
327
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if "top_logprobs" in data and data["top_logprobs"] is not None:
            if "logprobs" not in data or data["logprobs"] is False:
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )
328
            elif data["top_logprobs"] < 0:
329
                raise ValueError(
330
                    "`top_logprobs` must be a value a positive value.")
331
332
        return data

Zhuohan Li's avatar
Zhuohan Li committed
333

334
class CompletionRequest(OpenAIBaseModel):
335
336
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
Zhuohan Li's avatar
Zhuohan Li committed
337
    model: str
338
    prompt: Union[List[int], List[List[int]], str, List[str]]
339
    best_of: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
340
341
342
    echo: Optional[bool] = False
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
343
344
    logprobs: Optional[int] = None
    max_tokens: Optional[int] = 16
345
    n: int = 1
346
    presence_penalty: Optional[float] = 0.0
347
    seed: Optional[int] = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
348
349
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
    stream: Optional[bool] = False
350
    stream_options: Optional[StreamOptions] = None
351
352
353
    suffix: Optional[str] = None
    temperature: Optional[float] = 1.0
    top_p: Optional[float] = 1.0
Zhuohan Li's avatar
Zhuohan Li committed
354
    user: Optional[str] = None
355
356

    # doc: begin-completion-sampling-params
357
358
359
360
361
362
    use_beam_search: bool = False
    top_k: int = -1
    min_p: float = 0.0
    repetition_penalty: float = 1.0
    length_penalty: float = 1.0
    early_stopping: bool = False
363
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
364
365
366
367
368
    include_stop_str_in_output: bool = False
    ignore_eos: bool = False
    min_tokens: int = 0
    skip_special_tokens: bool = True
    spaces_between_special_tokens: bool = True
369
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
370
    allowed_token_ids: Optional[List[int]] = None
371
372
373
    # doc: end-completion-sampling-params

    # doc: begin-completion-extra-params
374
375
    add_special_tokens: bool = Field(
        default=True,
376
        description=(
377
378
            "If true (the default), special tokens (e.g. BOS) will be added to "
            "the prompt."),
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
    )
    response_format: Optional[ResponseFormat] = Field(
        default=None,
        description=
        ("Similar to chat completion, this parameter specifies the format of "
         "output. Only {'type': 'json_object'} or {'type': 'text' } is "
         "supported."),
    )
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
        description=("If specified, the output will follow the JSON schema."),
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
406
407
408
409
410
411
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be one of "
            "'outlines' / 'lm-format-enforcer'"))
412
413
414
415
416
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
417
418

    # doc: end-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
419

420
421
422
423
424
425
426
427
    def to_sampling_params(
            self, tokenizer: PreTrainedTokenizer,
            guided_decode_logits_processor: Optional[LogitsProcessor],
            default_max_tokens: int) -> SamplingParams:
        max_tokens = self.max_tokens
        if max_tokens is None:
            max_tokens = default_max_tokens

428
429
        echo_without_generation = self.echo and self.max_tokens == 0

430
431
432
433
434
        logits_processors = get_logits_processors(
            logit_bias=self.logit_bias,
            allowed_token_ids=self.allowed_token_ids,
            tokenizer=tokenizer,
        )
435
436
        if guided_decode_logits_processor:
            logits_processors.append(guided_decode_logits_processor)
437

438
439
440
441
442
443
444
445
446
447
        return SamplingParams(
            n=self.n,
            best_of=self.best_of,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
            top_k=self.top_k,
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
448
            seed=self.seed,
449
450
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
451
            logprobs=self.logprobs,
452
            ignore_eos=self.ignore_eos,
453
            max_tokens=max_tokens if not echo_without_generation else 1,
454
            min_tokens=self.min_tokens,
455
            use_beam_search=self.use_beam_search,
456
            early_stopping=self.early_stopping,
457
458
            prompt_logprobs=self.logprobs if self.echo else None,
            skip_special_tokens=self.skip_special_tokens,
459
            spaces_between_special_tokens=self.spaces_between_special_tokens,
460
461
            include_stop_str_in_output=self.include_stop_str_in_output,
            length_penalty=self.length_penalty,
462
            logits_processors=logits_processors,
463
            truncate_prompt_tokens=self.truncate_prompt_tokens,
464
465
        )

466
467
468
469
470
471
472
473
474
475
476
477
478
479
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
        return data

480
481
482
483
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if "logprobs" in data and data[
484
485
                "logprobs"] is not None and not data["logprobs"] >= 0:
            raise ValueError("if passed, `logprobs` must be a positive value.")
486
487
        return data

488
489
490
491
492
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
            raise ValueError(
493
                "Stream options can only be defined when stream is true.")
494
495
        return data

Zhuohan Li's avatar
Zhuohan Li committed
496

497
class EmbeddingRequest(OpenAIBaseModel):
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
    model: str
    input: Union[List[int], List[List[int]], str, List[str]]
    encoding_format: Optional[str] = Field('float', pattern='^(float|base64)$')
    dimensions: Optional[int] = None
    user: Optional[str] = None

    # doc: begin-embedding-pooling-params
    additional_data: Optional[Any] = None

    # doc: end-embedding-pooling-params

    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


515
class CompletionLogProbs(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
516
517
518
    text_offset: List[int] = Field(default_factory=list)
    token_logprobs: List[Optional[float]] = Field(default_factory=list)
    tokens: List[str] = Field(default_factory=list)
519
520
    top_logprobs: List[Optional[Dict[str,
                                     float]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
521
522


523
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
524
525
    index: int
    text: str
526
    logprobs: Optional[CompletionLogProbs] = None
527
528
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
529
530
531
532
533
534
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
Zhuohan Li's avatar
Zhuohan Li committed
535
536


537
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
538
539
540
541
542
543
544
545
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseChoice]
    usage: UsageInfo


546
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
547
548
    index: int
    text: str
549
    logprobs: Optional[CompletionLogProbs] = None
550
551
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
552
553
554
555
556
557
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
Zhuohan Li's avatar
Zhuohan Li committed
558
559


560
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
561
562
563
564
565
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseStreamChoice]
566
    usage: Optional[UsageInfo] = Field(default=None)
567
568


569
class EmbeddingResponseData(OpenAIBaseModel):
570
571
    index: int
    object: str = "embedding"
572
    embedding: Union[List[float], str]
573
574


575
class EmbeddingResponse(OpenAIBaseModel):
576
577
578
579
580
581
582
583
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    data: List[EmbeddingResponseData]
    usage: UsageInfo


584
585
586
587
588
589
590
591
592
593
594
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    function: FunctionCall


595
class ChatMessage(OpenAIBaseModel):
596
597
    role: str
    content: str
598
    tool_calls: List[ToolCall] = Field(default_factory=list)
599
600


601
602
603
604
605
606
607
608
609
610
611
612
613
614
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
    bytes: Optional[List[int]] = None


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
    top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list)


class ChatCompletionLogProbs(OpenAIBaseModel):
    content: Optional[List[ChatCompletionLogProbsContent]] = None


615
class ChatCompletionResponseChoice(OpenAIBaseModel):
616
617
    index: int
    message: ChatMessage
618
    logprobs: Optional[ChatCompletionLogProbs] = None
619
    finish_reason: Optional[str] = None
620
    stop_reason: Optional[Union[int, str]] = None
621
622


623
class ChatCompletionResponse(OpenAIBaseModel):
624
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
625
    object: Literal["chat.completion"] = "chat.completion"
626
627
628
629
630
631
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseChoice]
    usage: UsageInfo


632
class DeltaMessage(OpenAIBaseModel):
633
634
    role: Optional[str] = None
    content: Optional[str] = None
635
    tool_calls: List[ToolCall] = Field(default_factory=list)
636
637


638
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
639
640
    index: int
    delta: DeltaMessage
641
    logprobs: Optional[ChatCompletionLogProbs] = None
642
    finish_reason: Optional[str] = None
643
    stop_reason: Optional[Union[int, str]] = None
644
645


646
class ChatCompletionStreamResponse(OpenAIBaseModel):
647
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
648
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
649
650
651
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseStreamChoice]
652
    usage: Optional[UsageInfo] = Field(default=None)
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673


class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

674
    # The parameters of the request.
675
    body: Union[ChatCompletionRequest, EmbeddingRequest]
676
677


678
679
680
681
682
683
684
685
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
686
    body: Optional[Union[ChatCompletionResponse, EmbeddingResponse]] = None
687
688


689
690
691
692
693
694
695
696
697
698
699
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

700
    response: Optional[BatchResponseData]
701
702
703
704

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
    error: Optional[Any]
705
706


707
708
709
710
711
712
713
714
715
716
717
class TokenizeCompletionRequest(OpenAIBaseModel):
    model: str
    prompt: str

    add_special_tokens: bool = Field(default=True)


class TokenizeChatRequest(OpenAIBaseModel):
    model: str
    messages: List[ChatCompletionMessageParam]

718
719
    add_generation_prompt: bool = Field(default=True)
    add_special_tokens: bool = Field(default=False)
720
721
722


TokenizeRequest = Union[TokenizeCompletionRequest, TokenizeChatRequest]
723
724
725
726
727


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
728
    tokens: List[int]
729
730
731
732
733
734
735
736
737


class DetokenizeRequest(OpenAIBaseModel):
    model: str
    tokens: List[int]


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str