protocol.py 25.7 KB
Newer Older
1
2
# Adapted from
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
Zhuohan Li's avatar
Zhuohan Li committed
3
import time
4
from typing import Any, Dict, List, Literal, Optional, Union
Zhuohan Li's avatar
Zhuohan Li committed
5

6
import torch
7
from pydantic import BaseModel, ConfigDict, Field, model_validator
8
from typing_extensions import Annotated
Zhuohan Li's avatar
Zhuohan Li committed
9

10
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
11
from vllm.pooling_params import PoolingParams
12
from vllm.sampling_params import SamplingParams
13
from vllm.utils import random_uuid
14

Zhuohan Li's avatar
Zhuohan Li committed
15

16
17
18
19
20
21
class OpenAIBaseModel(BaseModel):
    # OpenAI API does not allow extra fields
    model_config = ConfigDict(extra="forbid")


class ErrorResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
22
23
24
25
    object: str = "error"
    message: str
    type: str
    param: Optional[str] = None
26
    code: int
Zhuohan Li's avatar
Zhuohan Li committed
27
28


29
class ModelPermission(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
30
31
32
33
34
35
36
37
38
39
40
    id: str = Field(default_factory=lambda: f"modelperm-{random_uuid()}")
    object: str = "model_permission"
    created: int = Field(default_factory=lambda: int(time.time()))
    allow_create_engine: bool = False
    allow_sampling: bool = True
    allow_logprobs: bool = True
    allow_search_indices: bool = False
    allow_view: bool = True
    allow_fine_tuning: bool = False
    organization: str = "*"
    group: Optional[str] = None
41
    is_blocking: bool = False
Zhuohan Li's avatar
Zhuohan Li committed
42
43


44
class ModelCard(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
45
46
47
    id: str
    object: str = "model"
    created: int = Field(default_factory=lambda: int(time.time()))
Woosuk Kwon's avatar
Woosuk Kwon committed
48
    owned_by: str = "vllm"
Zhuohan Li's avatar
Zhuohan Li committed
49
50
    root: Optional[str] = None
    parent: Optional[str] = None
51
    max_model_len: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
52
53
54
    permission: List[ModelPermission] = Field(default_factory=list)


55
class ModelList(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
56
57
58
59
    object: str = "list"
    data: List[ModelCard] = Field(default_factory=list)


60
class UsageInfo(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
61
62
63
64
65
    prompt_tokens: int = 0
    total_tokens: int = 0
    completion_tokens: Optional[int] = 0


66
class ResponseFormat(OpenAIBaseModel):
67
    # type must be "json_object" or "text"
68
    type: Literal["text", "json_object"]
69
70


71
class StreamOptions(OpenAIBaseModel):
72
73
    include_usage: Optional[bool] = True
    continuous_usage_stats: Optional[bool] = True
74
75


76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
class FunctionDefinition(OpenAIBaseModel):
    name: str
    description: Optional[str] = None
    parameters: Optional[Dict[str, Any]] = None


class ChatCompletionToolsParam(OpenAIBaseModel):
    type: Literal["function"] = "function"
    function: FunctionDefinition


class ChatCompletionNamedFunction(OpenAIBaseModel):
    name: str


class ChatCompletionNamedToolChoiceParam(OpenAIBaseModel):
    function: ChatCompletionNamedFunction
    type: Literal["function"] = "function"


96
class ChatCompletionRequest(OpenAIBaseModel):
97
98
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/chat/create
99
    messages: List[ChatCompletionMessageParam]
100
101
102
103
    model: str
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
    logprobs: Optional[bool] = False
104
    top_logprobs: Optional[int] = 0
105
    max_tokens: Optional[int] = None
106
107
108
    n: Optional[int] = 1
    presence_penalty: Optional[float] = 0.0
    response_format: Optional[ResponseFormat] = None
109
110
111
    seed: Optional[int] = Field(None,
                                ge=torch.iinfo(torch.long).min,
                                le=torch.iinfo(torch.long).max)
112
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
113
    stream: Optional[bool] = False
114
    stream_options: Optional[StreamOptions] = None
115
116
    temperature: Optional[float] = 0.7
    top_p: Optional[float] = 1.0
117
118
119
    tools: Optional[List[ChatCompletionToolsParam]] = None
    tool_choice: Optional[Union[Literal["none"],
                                ChatCompletionNamedToolChoiceParam]] = "none"
Zhuohan Li's avatar
Zhuohan Li committed
120
    user: Optional[str] = None
121
122

    # doc: begin-chat-completion-sampling-params
123
124
    best_of: Optional[int] = None
    use_beam_search: Optional[bool] = False
125
126
127
128
    top_k: Optional[int] = -1
    min_p: Optional[float] = 0.0
    repetition_penalty: Optional[float] = 1.0
    length_penalty: Optional[float] = 1.0
129
    early_stopping: Optional[bool] = False
130
    ignore_eos: Optional[bool] = False
131
    min_tokens: Optional[int] = 0
132
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
133
    skip_special_tokens: Optional[bool] = True
134
    spaces_between_special_tokens: Optional[bool] = True
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
    # doc: end-chat-completion-sampling-params

    # doc: begin-chat-completion-extra-params
    echo: Optional[bool] = Field(
        default=False,
        description=(
            "If true, the new message will be prepended with the last message "
            "if they belong to the same role."),
    )
    add_generation_prompt: Optional[bool] = Field(
        default=True,
        description=
        ("If true, the generation prompt will be added to the chat template. "
         "This is a parameter used by chat template in tokenizer config of the "
         "model."),
    )
151
152
153
154
155
156
157
158
159
    add_special_tokens: Optional[bool] = Field(
        default=False,
        description=(
            "If true, special tokens (e.g. BOS) will be added to the prompt "
            "on top of what is added by the chat template. "
            "For most models, the chat template takes care of adding the "
            "special tokens so this should be set to False (as is the "
            "default)."),
    )
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
    documents: Optional[List[Dict[str, str]]] = Field(
        default=None,
        description=
        ("A list of dicts representing documents that will be accessible to "
         "the model if it is performing RAG (retrieval-augmented generation)."
         " If the template does not support RAG, this argument will have no "
         "effect. We recommend that each document should be a dict containing "
         "\"title\" and \"text\" keys."),
    )
    chat_template: Optional[str] = Field(
        default=None,
        description=(
            "A Jinja template to use for this conversion. "
            "If this is not passed, the model's default chat template will be "
            "used instead."),
    )
    chat_template_kwargs: Optional[Dict[str, Any]] = Field(
        default=None,
        description=("Additional kwargs to pass to the template renderer. "
                     "Will be accessible by the chat template."),
    )
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
    include_stop_str_in_output: Optional[bool] = Field(
        default=False,
        description=(
            "Whether to include the stop string in the output. "
            "This is only applied when the stop or stop_token_ids is set."),
    )
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
        description=("If specified, the output will follow the JSON schema."),
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
206
207
208
209
210
211
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be either "
            "'outlines' / 'lm-format-enforcer'"))
212
213
214
215
216
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
217
218

    # doc: end-chat-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
219

220
    def to_sampling_params(self) -> SamplingParams:
221
        # We now allow logprobs being true without top_logrobs.
222
223
224

        logits_processors = None
        if self.logit_bias:
225
226
227
228
229
230
231
232
233
234
            logit_bias: Dict[int, float] = {}
            try:
                for token_id, bias in self.logit_bias.items():
                    # Convert token_id to integer before we add to LLMEngine
                    # Clamp the bias between -100 and 100 per OpenAI API spec
                    logit_bias[int(token_id)] = min(100, max(-100, bias))
            except ValueError as exc:
                raise ValueError(f"Found token_id `{token_id}` in logit_bias "
                                 f"but token_id must be an integer or string "
                                 f"representing an integer") from exc
235
236
237
238

            def logit_bias_logits_processor(
                    token_ids: List[int],
                    logits: torch.Tensor) -> torch.Tensor:
239
240
                for token_id, bias in logit_bias.items():
                    logits[token_id] += bias
241
242
243
244
                return logits

            logits_processors = [logit_bias_logits_processor]

245
246
247
248
249
250
251
252
        return SamplingParams(
            n=self.n,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
253
            seed=self.seed,
254
255
256
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
            max_tokens=self.max_tokens,
257
            min_tokens=self.min_tokens,
258
259
            logprobs=self.top_logprobs if self.logprobs else None,
            prompt_logprobs=self.top_logprobs if self.echo else None,
260
261
262
263
            best_of=self.best_of,
            top_k=self.top_k,
            ignore_eos=self.ignore_eos,
            use_beam_search=self.use_beam_search,
264
            early_stopping=self.early_stopping,
265
266
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=self.spaces_between_special_tokens,
267
268
            include_stop_str_in_output=self.include_stop_str_in_output,
            length_penalty=self.length_penalty,
269
            logits_processors=logits_processors,
270
271
        )

272
273
274
275
276
277
278
279
280
    @model_validator(mode='before')
    @classmethod
    def validate_stream_options(cls, values):
        if (values.get('stream_options') is not None
                and not values.get('stream')):
            raise ValueError(
                "stream_options can only be set if stream is true")
        return values

281
282
283
284
285
286
287
288
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
289
        # you can only use one kind of guided decoding
290
291
292
293
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
        # you can only either use guided decoding or tools, not both
        if guide_count > 1 and "tool_choice" in data and data[
                "tool_choice"] != "none":
            raise ValueError(
                "You can only either use guided decoding or tools, not both.")
        return data

    @model_validator(mode="before")
    @classmethod
    def check_tool_choice(cls, data):
        if "tool_choice" in data and data["tool_choice"] != "none":
            if not isinstance(data["tool_choice"], dict):
                raise ValueError("Currently only named tools are supported.")
            if "tools" not in data or data["tools"] is None:
                raise ValueError(
                    "When using `tool_choice`, `tools` must be set.")
310
311
        return data

312
313
314
315
316
317
318
319
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if "top_logprobs" in data and data["top_logprobs"] is not None:
            if "logprobs" not in data or data["logprobs"] is False:
                raise ValueError(
                    "when using `top_logprobs`, `logprobs` must be set to true."
                )
320
            elif data["top_logprobs"] < 0:
321
                raise ValueError(
322
                    "`top_logprobs` must be a value a positive value.")
323
324
        return data

Zhuohan Li's avatar
Zhuohan Li committed
325

326
class CompletionRequest(OpenAIBaseModel):
327
328
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/completions/create
Zhuohan Li's avatar
Zhuohan Li committed
329
    model: str
330
    prompt: Union[List[int], List[List[int]], str, List[str]]
331
    best_of: Optional[int] = None
Zhuohan Li's avatar
Zhuohan Li committed
332
333
334
    echo: Optional[bool] = False
    frequency_penalty: Optional[float] = 0.0
    logit_bias: Optional[Dict[str, float]] = None
335
336
    logprobs: Optional[int] = None
    max_tokens: Optional[int] = 16
337
    n: int = 1
338
    presence_penalty: Optional[float] = 0.0
339
340
341
    seed: Optional[int] = Field(None,
                                ge=torch.iinfo(torch.long).min,
                                le=torch.iinfo(torch.long).max)
342
343
    stop: Optional[Union[str, List[str]]] = Field(default_factory=list)
    stream: Optional[bool] = False
344
    stream_options: Optional[StreamOptions] = None
345
346
347
    suffix: Optional[str] = None
    temperature: Optional[float] = 1.0
    top_p: Optional[float] = 1.0
Zhuohan Li's avatar
Zhuohan Li committed
348
    user: Optional[str] = None
349
350

    # doc: begin-completion-sampling-params
Zhuohan Li's avatar
Zhuohan Li committed
351
    use_beam_search: Optional[bool] = False
352
353
354
355
    top_k: Optional[int] = -1
    min_p: Optional[float] = 0.0
    repetition_penalty: Optional[float] = 1.0
    length_penalty: Optional[float] = 1.0
356
    early_stopping: Optional[bool] = False
357
    stop_token_ids: Optional[List[int]] = Field(default_factory=list)
358
    ignore_eos: Optional[bool] = False
359
    min_tokens: Optional[int] = 0
360
    skip_special_tokens: Optional[bool] = True
361
    spaces_between_special_tokens: Optional[bool] = True
362
    truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
    # doc: end-completion-sampling-params

    # doc: begin-completion-extra-params
    include_stop_str_in_output: Optional[bool] = Field(
        default=False,
        description=(
            "Whether to include the stop string in the output. "
            "This is only applied when the stop or stop_token_ids is set."),
    )
    response_format: Optional[ResponseFormat] = Field(
        default=None,
        description=
        ("Similar to chat completion, this parameter specifies the format of "
         "output. Only {'type': 'json_object'} or {'type': 'text' } is "
         "supported."),
    )
    guided_json: Optional[Union[str, dict, BaseModel]] = Field(
        default=None,
        description=("If specified, the output will follow the JSON schema."),
    )
    guided_regex: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the regex pattern."),
    )
    guided_choice: Optional[List[str]] = Field(
        default=None,
        description=(
            "If specified, the output will be exactly one of the choices."),
    )
    guided_grammar: Optional[str] = Field(
        default=None,
        description=(
            "If specified, the output will follow the context free grammar."),
    )
398
399
400
401
402
403
    guided_decoding_backend: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default guided decoding backend "
            "of the server for this specific request. If set, must be one of "
            "'outlines' / 'lm-format-enforcer'"))
404
405
406
407
408
    guided_whitespace_pattern: Optional[str] = Field(
        default=None,
        description=(
            "If specified, will override the default whitespace pattern "
            "for guided json decoding."))
409
410

    # doc: end-completion-extra-params
Zhuohan Li's avatar
Zhuohan Li committed
411

412
413
414
    def to_sampling_params(self):
        echo_without_generation = self.echo and self.max_tokens == 0

415
416
        logits_processors = None
        if self.logit_bias:
417
418
419
420
421
422
423
424
425
426
            logit_bias: Dict[int, float] = {}
            try:
                for token_id, bias in self.logit_bias.items():
                    # Convert token_id to integer
                    # Clamp the bias between -100 and 100 per OpenAI API spec
                    logit_bias[int(token_id)] = min(100, max(-100, bias))
            except ValueError as exc:
                raise ValueError(f"Found token_id `{token_id}` in logit_bias "
                                 f"but token_id must be an integer or string "
                                 f"representing an integer") from exc
427
428
429
430

            def logit_bias_logits_processor(
                    token_ids: List[int],
                    logits: torch.Tensor) -> torch.Tensor:
431
432
                for token_id, bias in logit_bias.items():
                    logits[token_id] += bias
433
434
435
436
                return logits

            logits_processors = [logit_bias_logits_processor]

437
438
439
440
441
442
443
444
445
446
        return SamplingParams(
            n=self.n,
            best_of=self.best_of,
            presence_penalty=self.presence_penalty,
            frequency_penalty=self.frequency_penalty,
            repetition_penalty=self.repetition_penalty,
            temperature=self.temperature,
            top_p=self.top_p,
            top_k=self.top_k,
            min_p=self.min_p,
Nick Hill's avatar
Nick Hill committed
447
            seed=self.seed,
448
449
450
451
            stop=self.stop,
            stop_token_ids=self.stop_token_ids,
            ignore_eos=self.ignore_eos,
            max_tokens=self.max_tokens if not echo_without_generation else 1,
452
            min_tokens=self.min_tokens,
453
454
            logprobs=self.logprobs,
            use_beam_search=self.use_beam_search,
455
            early_stopping=self.early_stopping,
456
457
458
            prompt_logprobs=self.logprobs if self.echo else None,
            skip_special_tokens=self.skip_special_tokens,
            spaces_between_special_tokens=(self.spaces_between_special_tokens),
459
460
            include_stop_str_in_output=self.include_stop_str_in_output,
            length_penalty=self.length_penalty,
461
            logits_processors=logits_processors,
462
            truncate_prompt_tokens=self.truncate_prompt_tokens,
463
464
        )

465
466
467
468
469
470
471
472
473
474
475
476
477
478
    @model_validator(mode="before")
    @classmethod
    def check_guided_decoding_count(cls, data):
        guide_count = sum([
            "guided_json" in data and data["guided_json"] is not None,
            "guided_regex" in data and data["guided_regex"] is not None,
            "guided_choice" in data and data["guided_choice"] is not None
        ])
        if guide_count > 1:
            raise ValueError(
                "You can only use one kind of guided decoding "
                "('guided_json', 'guided_regex' or 'guided_choice').")
        return data

479
480
481
482
    @model_validator(mode="before")
    @classmethod
    def check_logprobs(cls, data):
        if "logprobs" in data and data[
483
484
                "logprobs"] is not None and not data["logprobs"] >= 0:
            raise ValueError("if passed, `logprobs` must be a positive value.")
485
486
        return data

487
488
489
490
491
492
493
494
    @model_validator(mode="before")
    @classmethod
    def validate_stream_options(cls, data):
        if data.get("stream_options") and not data.get("stream"):
            raise ValueError(
                "Stream options can only be defined when stream is True.")
        return data

Zhuohan Li's avatar
Zhuohan Li committed
495

496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
class EmbeddingRequest(BaseModel):
    # Ordered by official OpenAI API documentation
    # https://platform.openai.com/docs/api-reference/embeddings
    model: str
    input: Union[List[int], List[List[int]], str, List[str]]
    encoding_format: Optional[str] = Field('float', pattern='^(float|base64)$')
    dimensions: Optional[int] = None
    user: Optional[str] = None

    # doc: begin-embedding-pooling-params
    additional_data: Optional[Any] = None

    # doc: end-embedding-pooling-params

    def to_pooling_params(self):
        return PoolingParams(additional_data=self.additional_data)


514
class CompletionLogProbs(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
515
516
517
    text_offset: List[int] = Field(default_factory=list)
    token_logprobs: List[Optional[float]] = Field(default_factory=list)
    tokens: List[str] = Field(default_factory=list)
518
519
    top_logprobs: List[Optional[Dict[str,
                                     float]]] = Field(default_factory=list)
Zhuohan Li's avatar
Zhuohan Li committed
520
521


522
class CompletionResponseChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
523
524
    index: int
    text: str
525
    logprobs: Optional[CompletionLogProbs] = None
526
527
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
528
529
530
531
532
533
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
Zhuohan Li's avatar
Zhuohan Li committed
534
535


536
class CompletionResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
537
538
539
540
541
542
543
544
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseChoice]
    usage: UsageInfo


545
class CompletionResponseStreamChoice(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
546
547
    index: int
    text: str
548
    logprobs: Optional[CompletionLogProbs] = None
549
550
    finish_reason: Optional[str] = None
    stop_reason: Optional[Union[int, str]] = Field(
551
552
553
554
555
556
        default=None,
        description=(
            "The stop string or token id that caused the completion "
            "to stop, None if the completion finished for some other reason "
            "including encountering the EOS token"),
    )
Zhuohan Li's avatar
Zhuohan Li committed
557
558


559
class CompletionStreamResponse(OpenAIBaseModel):
Zhuohan Li's avatar
Zhuohan Li committed
560
561
562
563
564
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "text_completion"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[CompletionResponseStreamChoice]
565
    usage: Optional[UsageInfo] = Field(default=None)
566
567


568
569
570
class EmbeddingResponseData(BaseModel):
    index: int
    object: str = "embedding"
571
    embedding: Union[List[float], str]
572
573
574
575
576
577
578
579
580
581
582


class EmbeddingResponse(BaseModel):
    id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
    object: str = "list"
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    data: List[EmbeddingResponseData]
    usage: UsageInfo


583
584
585
586
587
588
589
590
591
592
593
class FunctionCall(OpenAIBaseModel):
    name: str
    arguments: str


class ToolCall(OpenAIBaseModel):
    id: str = Field(default_factory=lambda: f"chatcmpl-tool-{random_uuid()}")
    type: Literal["function"] = "function"
    function: FunctionCall


594
class ChatMessage(OpenAIBaseModel):
595
596
    role: str
    content: str
597
    tool_calls: List[ToolCall] = Field(default_factory=list)
598
599


600
601
602
603
604
605
606
607
608
609
610
611
612
613
class ChatCompletionLogProb(OpenAIBaseModel):
    token: str
    logprob: float = -9999.0
    bytes: Optional[List[int]] = None


class ChatCompletionLogProbsContent(ChatCompletionLogProb):
    top_logprobs: List[ChatCompletionLogProb] = Field(default_factory=list)


class ChatCompletionLogProbs(OpenAIBaseModel):
    content: Optional[List[ChatCompletionLogProbsContent]] = None


614
class ChatCompletionResponseChoice(OpenAIBaseModel):
615
616
    index: int
    message: ChatMessage
617
    logprobs: Optional[ChatCompletionLogProbs] = None
618
    finish_reason: Optional[str] = None
619
    stop_reason: Optional[Union[int, str]] = None
620
621


622
class ChatCompletionResponse(OpenAIBaseModel):
623
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
624
    object: Literal["chat.completion"] = "chat.completion"
625
626
627
628
629
630
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseChoice]
    usage: UsageInfo


631
class DeltaMessage(OpenAIBaseModel):
632
633
    role: Optional[str] = None
    content: Optional[str] = None
634
    tool_calls: List[ToolCall] = Field(default_factory=list)
635
636


637
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
638
639
    index: int
    delta: DeltaMessage
640
    logprobs: Optional[ChatCompletionLogProbs] = None
641
    finish_reason: Optional[str] = None
642
    stop_reason: Optional[Union[int, str]] = None
643
644


645
class ChatCompletionStreamResponse(OpenAIBaseModel):
646
    id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
647
    object: Literal["chat.completion.chunk"] = "chat.completion.chunk"
648
649
650
    created: int = Field(default_factory=lambda: int(time.time()))
    model: str
    choices: List[ChatCompletionResponseStreamChoice]
651
    usage: Optional[UsageInfo] = Field(default=None)
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676


class BatchRequestInput(OpenAIBaseModel):
    """
    The per-line object of the batch input file.

    NOTE: Currently only the `/v1/chat/completions` endpoint is supported.
    """

    # A developer-provided per-request id that will be used to match outputs to
    # inputs. Must be unique for each request in a batch.
    custom_id: str

    # The HTTP method to be used for the request. Currently only POST is
    # supported.
    method: str

    # The OpenAI API relative URL to be used for the request. Currently
    # /v1/chat/completions is supported.
    url: str

    # The parameteters of the request.
    body: Union[ChatCompletionRequest, ]


677
678
679
680
681
682
683
684
class BatchResponseData(OpenAIBaseModel):
    # HTTP status code of the response.
    status_code: int = 200

    # An unique identifier for the API request.
    request_id: str

    # The body of the response.
685
    body: Optional[ChatCompletionResponse] = None
686
687


688
689
690
691
692
693
694
695
696
697
698
class BatchRequestOutput(OpenAIBaseModel):
    """
    The per-line object of the batch output and error files
    """

    id: str

    # A developer-provided per-request id that will be used to match outputs to
    # inputs.
    custom_id: str

699
    response: Optional[BatchResponseData]
700
701
702
703

    # For requests that failed with a non-HTTP error, this will contain more
    # information on the cause of the failure.
    error: Optional[Any]
704
705
706


class TokenizeRequest(OpenAIBaseModel):
707
708
709
710
    add_generation_prompt: bool = Field(default=True)
    add_special_tokens: bool = Field(default=False)
    prompt: Optional[str] = Field(default=None)
    messages: Optional[List[ChatCompletionMessageParam]] = Field(default=None)
711
712
713
714
715
716
    model: str


class TokenizeResponse(OpenAIBaseModel):
    count: int
    max_model_len: int
717
    tokens: List[int]
718
719
720
721
722
723
724
725
726


class DetokenizeRequest(OpenAIBaseModel):
    model: str
    tokens: List[int]


class DetokenizeResponse(OpenAIBaseModel):
    prompt: str