sampling_params.py 25.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""Sampling parameters for text generation."""
4
import copy
5
import warnings
6
from dataclasses import field
7
from enum import Enum, IntEnum
8
from functools import cached_property
9
from typing import Annotated, Any, Optional, Union
10

11
import msgspec
12
from pydantic.dataclasses import dataclass
Woosuk Kwon's avatar
Woosuk Kwon committed
13

14
from vllm.logger import init_logger
15
from vllm.logits_process import LogitsProcessor
16
from vllm.transformers_utils.tokenizer import AnyTokenizer
17
18
19

logger = init_logger(__name__)

20
_SAMPLING_EPS = 1e-5
21
_MAX_TEMP = 1e-2
Woosuk Kwon's avatar
Woosuk Kwon committed
22

23

24
25
26
class SamplingType(IntEnum):
    GREEDY = 0
    RANDOM = 1
Nick Hill's avatar
Nick Hill committed
27
    RANDOM_SEED = 2
28
29


30
31
# maybe make msgspec?
@dataclass
32
33
class StructuredOutputsParams:
    # One of these fields will be used to build a logit processor.
34
    json: Optional[Union[str, dict]] = None
35
    regex: Optional[str] = None
36
    choice: Optional[list[str]] = None
37
38
    grammar: Optional[str] = None
    json_object: Optional[bool] = None
39
    # These are other options that can be set.
40
41
42
    disable_fallback: bool = False
    disable_any_whitespace: bool = False
    disable_additional_properties: bool = False
43
    whitespace_pattern: Optional[str] = None
44
    structural_tag: Optional[str] = None
45

46
47
48
49
    _backend: Optional[str] = field(default=None, init=False)
    """CAUTION: Should only be set by Processor._validate_structured_output"""
    _backend_was_auto: bool = field(default=False, init=False)
    """CAUTION: Should only be set by Processor._validate_structured_output"""
50
51
52

    def __post_init__(self):
        """Validate that some fields are mutually exclusive."""
53
        count = sum([
54
55
56
            self.json is not None, self.regex is not None, self.choice
            is not None, self.grammar is not None, self.json_object is not None
        ])
57
        if count > 1:
58
            raise ValueError(
59
60
                "You can only use one kind of structured outputs constraint "
                f"but multiple are specified: {self.__dict__}")
61
62


63
64
65
66
67
68
69
70
71
72
73
74
75
@dataclass
class GuidedDecodingParams(StructuredOutputsParams):

    def __post_init__(self):
        warnings.warn(
            "GuidedDecodingParams is deprecated. This will be removed in "
            "v0.12.0 or v1.0.0, which ever is soonest. Please use "
            "StructuredOutputsParams instead.",
            DeprecationWarning,
            stacklevel=2)
        return super().__post_init__()


76
77
78
79
80
class RequestOutputKind(Enum):
    # Return entire output so far in every RequestOutput
    CUMULATIVE = 0
    # Return only deltas in each RequestOutput
    DELTA = 1
81
    # Do not return intermediate RequestOutput
82
83
84
    FINAL_ONLY = 2


85
86
87
88
89
class SamplingParams(
        msgspec.Struct,
        omit_defaults=True,  # type: ignore[call-arg]
        # required for @cached_property.
        dict=True):  # type: ignore[call-arg]
90
91
92
93
94
95
    """Sampling parameters for text generation.

    Overall, we follow the sampling parameters from the OpenAI text completion
    API (https://platform.openai.com/docs/api-reference/completions/create).
    In addition, we support beam search, which is not supported by OpenAI.
    """
Woosuk Kwon's avatar
Woosuk Kwon committed
96

97
    n: int = 1
98
99
100
101
102
103
104
    """Number of outputs to return for the given prompt request.

    NOTE:
        `AsyncLLM` streams outputs by default. When `n > 1`, all `n` outputs
        are generated and streamed cumulatively per request. To see all `n`
        outputs upon completion, use `output_kind=RequestOutputKind.FINAL_ONLY`
        in `SamplingParams`."""
105
    best_of: Optional[int] = None
106
107
108
109
    """Number of output sequences that are generated from the prompt. From
    these `best_of` sequences, the top `n` sequences are returned. `best_of`
    must be greater than or equal to `n`. By default, `best_of` is set to `n`.
    Warning, this is only supported in V0."""
110
    _real_n: Optional[int] = None
111
    presence_penalty: float = 0.0
112
113
114
    """Penalizes new tokens based on whether they appear in the generated text
    so far. Values > 0 encourage the model to use new tokens, while values < 0
    encourage the model to repeat tokens."""
115
    frequency_penalty: float = 0.0
116
117
118
    """Penalizes new tokens based on their frequency in the generated text so
    far. Values > 0 encourage the model to use new tokens, while values < 0
    encourage the model to repeat tokens."""
119
    repetition_penalty: float = 1.0
120
121
122
    """Penalizes new tokens based on whether they appear in the prompt and the
    generated text so far. Values > 1 encourage the model to use new tokens,
    while values < 1 encourage the model to repeat tokens."""
123
    temperature: float = 1.0
124
125
126
    """Controls the randomness of the sampling. Lower values make the model
    more deterministic, while higher values make the model more random. Zero
    means greedy sampling."""
127
    top_p: float = 1.0
128
129
    """Controls the cumulative probability of the top tokens to consider. Must
    be in (0, 1]. Set to 1 to consider all tokens."""
130
    top_k: int = 0
131
132
    """Controls the number of top tokens to consider. Set to 0 (or -1) to
    consider all tokens."""
133
    min_p: float = 0.0
134
135
136
    """Represents the minimum probability for a token to be considered,
    relative to the probability of the most likely token. Must be in [0, 1].
    Set to 0 to disable this."""
137
    seed: Optional[int] = None
138
    """Random seed to use for the generation."""
139
    stop: Optional[Union[str, list[str]]] = None
140
141
    """String(s) that stop the generation when they are generated. The returned
    output will not contain the stop strings."""
142
    stop_token_ids: Optional[list[int]] = None
143
144
145
    """Token IDs that stop the generation when they are generated. The returned
    output will contain the stop tokens unless the stop tokens are special
    tokens."""
146
    ignore_eos: bool = False
147
148
    """Whether to ignore the EOS token and continue generating
    tokens after the EOS token is generated."""
149
    max_tokens: Optional[int] = 16
150
    """Maximum number of tokens to generate per output sequence."""
151
    min_tokens: int = 0
152
153
    """Minimum number of tokens to generate per output sequence before EOS or
    `stop_token_ids` can be generated"""
154
    logprobs: Optional[int] = None
155
156
157
158
159
160
161
    """Number of log probabilities to return per output token. When set to
    `None`, no probability is returned. If set to a non-`None` value, the
    result includes the log probabilities of the specified number of most
    likely tokens, as well as the chosen tokens. Note that the implementation
    follows the OpenAI API: The API will always return the log probability of
    the sampled token, so there may be up to `logprobs+1` elements in the
    response. When set to -1, return all `vocab_size` log probabilities."""
162
    prompt_logprobs: Optional[int] = None
163
164
    """Number of log probabilities to return per prompt token.
    When set to -1, return all `vocab_size` log probabilities."""
165
166
167
168
    # NOTE: This parameter is only exposed at the engine level for now.
    # It is not exposed in the OpenAI API server, as the OpenAI API does
    # not support returning only a list of token IDs.
    detokenize: bool = True
169
    """Whether to detokenize the output."""
170
    skip_special_tokens: bool = True
171
    """Whether to skip special tokens in the output."""
172
    spaces_between_special_tokens: bool = True
173
    """Whether to add spaces between special tokens in the output."""
174
175
    # Optional[list[LogitsProcessor]] type. We use Any here because
    # Optional[list[LogitsProcessor]] type is not supported by msgspec.
176
    logits_processors: Optional[Any] = None
177
178
    """Functions that modify logits based on previously generated tokens, and
    optionally prompt tokens as a first argument."""
179
    include_stop_str_in_output: bool = False
180
    """Whether to include the stop strings in output text."""
181
182
    truncate_prompt_tokens: Optional[Annotated[int,
                                               msgspec.Meta(ge=-1)]] = None
183
184
185
    """If set to -1, will use the truncation size supported by the model. If
    set to an integer k, will use only the last k tokens from the prompt
    (i.e., left truncation). If set to `None`, truncation is disabled."""
186
    output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
187
188
189
190

    # The below fields are not supposed to be used as an input.
    # They are set in post_init.
    output_text_buffer_length: int = 0
191
    _all_stop_token_ids: set[int] = msgspec.field(default_factory=set)
192

193
    # Fields used to construct logits processors
194
195
    structured_outputs: Optional[StructuredOutputsParams] = None
    """Parameters for configuring structured outputs."""
196
197
    guided_decoding: Optional[GuidedDecodingParams] = None
    """Deprecated alias for structured_outputs."""
198
    logit_bias: Optional[dict[int, float]] = None
199
200
    """If provided, the engine will construct a logits processor that applies
    these logit biases."""
201
    allowed_token_ids: Optional[list[int]] = None
202
203
    """If provided, the engine will construct a logits processor which only
    retains scores for the given token ids."""
204
    extra_args: Optional[dict[str, Any]] = None
205
206
207
    """Arbitrary additional args, that can be used by custom sampling
    implementations, plugins, etc. Not used by any in-tree sampling
    implementations."""
208

209
210
    # Fields used for bad words
    bad_words: Optional[list[str]] = None
211
212
213
    """Words that are not allowed to be generated. More precisely, only the
    last token of a corresponding token sequence is not allowed when the next
    generated token can complete the sequence."""
214
    _bad_words_token_ids: Optional[list[list[int]]] = None
215

216
217
218
    @staticmethod
    def from_optional(
        n: Optional[int] = 1,
219
        best_of: Optional[int] = None,
220
221
222
223
224
        presence_penalty: Optional[float] = 0.0,
        frequency_penalty: Optional[float] = 0.0,
        repetition_penalty: Optional[float] = 1.0,
        temperature: Optional[float] = 1.0,
        top_p: Optional[float] = 1.0,
225
        top_k: int = 0,
226
227
        min_p: float = 0.0,
        seed: Optional[int] = None,
228
229
230
        stop: Optional[Union[str, list[str]]] = None,
        stop_token_ids: Optional[list[int]] = None,
        bad_words: Optional[list[str]] = None,
231
232
233
234
235
236
237
238
239
        include_stop_str_in_output: bool = False,
        ignore_eos: bool = False,
        max_tokens: Optional[int] = 16,
        min_tokens: int = 0,
        logprobs: Optional[int] = None,
        prompt_logprobs: Optional[int] = None,
        detokenize: bool = True,
        skip_special_tokens: bool = True,
        spaces_between_special_tokens: bool = True,
240
        logits_processors: Optional[list[LogitsProcessor]] = None,
241
        truncate_prompt_tokens: Optional[Annotated[int,
242
243
                                                   msgspec.Meta(
                                                       ge=-1)]] = None,
244
        output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
245
        structured_outputs: Optional[StructuredOutputsParams] = None,
246
        guided_decoding: Optional[GuidedDecodingParams] = None,
247
248
        logit_bias: Optional[Union[dict[int, float], dict[str, float]]] = None,
        allowed_token_ids: Optional[list[int]] = None,
249
        extra_args: Optional[dict[str, Any]] = None,
250
    ) -> "SamplingParams":
251
        if logit_bias is not None:
252
253
            # Convert token_id to integer
            # Clamp the bias between -100 and 100 per OpenAI API spec
254
            logit_bias = {
255
                int(token): min(100.0, max(-100.0, bias))
256
257
                for token, bias in logit_bias.items()
            }
258
259
260
261
262
263
264
265
266
        if guided_decoding is not None:
            warnings.warn(
                "guided_decoding is deprecated. This will be removed in "
                "v0.12.0 or v1.0.0, which ever is soonest. Please use "
                "structured_outputs instead.",
                DeprecationWarning,
                stacklevel=2)
            structured_outputs = guided_decoding
            guided_decoding = None
267

268
269
        return SamplingParams(
            n=1 if n is None else n,
270
            best_of=best_of,
271
272
273
274
275
276
277
278
279
280
281
282
283
            presence_penalty=0.0
            if presence_penalty is None else presence_penalty,
            frequency_penalty=0.0
            if frequency_penalty is None else frequency_penalty,
            repetition_penalty=1.0
            if repetition_penalty is None else repetition_penalty,
            temperature=1.0 if temperature is None else temperature,
            top_p=1.0 if top_p is None else top_p,
            top_k=top_k,
            min_p=min_p,
            seed=seed,
            stop=stop,
            stop_token_ids=stop_token_ids,
284
            bad_words=bad_words,
285
286
287
288
289
290
291
292
293
294
295
            include_stop_str_in_output=include_stop_str_in_output,
            ignore_eos=ignore_eos,
            max_tokens=max_tokens,
            min_tokens=min_tokens,
            logprobs=logprobs,
            prompt_logprobs=prompt_logprobs,
            detokenize=detokenize,
            skip_special_tokens=skip_special_tokens,
            spaces_between_special_tokens=spaces_between_special_tokens,
            logits_processors=logits_processors,
            truncate_prompt_tokens=truncate_prompt_tokens,
296
            output_kind=output_kind,
297
            structured_outputs=structured_outputs,
298
299
            logit_bias=logit_bias,
            allowed_token_ids=allowed_token_ids,
300
            extra_args=extra_args,
301
302
        )

303
    def __post_init__(self) -> None:
304
305
306
307
308
309
310
311
312
313
314
315
316
317
        # how we deal with `best_of``:
        # if `best_of`` is not set, we default to `n`;
        # if `best_of`` is set, we set `n`` to `best_of`,
        # and set `_real_n`` to the original `n`.
        # when we return the result, we will check
        # if we need to return `n` or `_real_n` results
        if self.best_of:
            if self.best_of < self.n:
                raise ValueError(
                    f"best_of must be greater than or equal to n, "
                    f"got n={self.n} and best_of={self.best_of}.")
            if not self._real_n:
                self._real_n = self.n
                self.n = self.best_of
318

319
        if 0 < self.temperature < _MAX_TEMP:
320
321
322
            logger.warning(
                "temperature %s is less than %s, which may cause numerical "
                "errors nan or inf in tensors. We have maxed it out to %s.",
323
324
                self.temperature, _MAX_TEMP, _MAX_TEMP)
            self.temperature = max(self.temperature, _MAX_TEMP)
325

326
        if self.seed == -1:
327
            self.seed = None
328

329
        if self.stop is None:
330
            self.stop = []
331
332
        elif isinstance(self.stop, str):
            self.stop = [self.stop]
333

334
        if self.stop_token_ids is None:
335
            self.stop_token_ids = []
336
337
338
339

        if self.bad_words is None:
            self.bad_words = []

340
341
342
343
344
        if self.logprobs is True:
            self.logprobs = 1

        if self.prompt_logprobs is True:
            self.prompt_logprobs = 1
345

346
347
        # Number of characters to hold back for stop string evaluation
        # until sequence is finished.
348
        if self.stop and not self.include_stop_str_in_output:
349
350
            self.output_text_buffer_length = max(len(s) for s in self.stop) - 1

351
        self._verify_args()
352
353
354
355

        if self.temperature < _SAMPLING_EPS:
            # Zero temperature means greedy sampling.
            self.top_p = 1.0
356
            self.top_k = 0
357
358
            self.min_p = 0.0
            self._verify_greedy_sampling()
359

360
        # eos_token_id is added to this by the engine
361
        self._all_stop_token_ids.update(self.stop_token_ids)
362

363
364
365
366
367
368
369
370
371
372
        if self.guided_decoding is not None:
            warnings.warn(
                "guided_decoding is deprecated. This will be removed in "
                "v0.12.0 or v1.0.0, which ever is soonest. Please use "
                "structured_outputs instead.",
                DeprecationWarning,
                stacklevel=2)
            self.structured_outputs = self.guided_decoding
            self.guided_decoding = None

373
    def _verify_args(self) -> None:
374
375
376
        if not isinstance(self.n, int):
            raise ValueError(f"n must be an int, but is of "
                             f"type {type(self.n)}")
377
378
        if self.n < 1:
            raise ValueError(f"n must be at least 1, got {self.n}.")
379
380
381
382
383
384
385
386
387
388
389
        if self.best_of is not None:
            if not isinstance(self.best_of, int):
                raise ValueError(
                    f"best_of must be an integer, got {type(self.best_of)}")
            if self.best_of < 1:
                raise ValueError(
                    f"best_of must be at least 1, got {self.best_of}")
            if self.best_of < self.n:
                raise ValueError(
                    f"best_of must be greater than or equal to n, "
                    f"got n={self.n} and best_of={self.best_of}.")
390
391
392
393
394
395
        if not -2.0 <= self.presence_penalty <= 2.0:
            raise ValueError("presence_penalty must be in [-2, 2], got "
                             f"{self.presence_penalty}.")
        if not -2.0 <= self.frequency_penalty <= 2.0:
            raise ValueError("frequency_penalty must be in [-2, 2], got "
                             f"{self.frequency_penalty}.")
396
397
398
399
        if self.repetition_penalty <= 0.0:
            raise ValueError(
                "repetition_penalty must be greater than zero, got "
                f"{self.repetition_penalty}.")
400
401
402
403
404
        if self.temperature < 0.0:
            raise ValueError(
                f"temperature must be non-negative, got {self.temperature}.")
        if not 0.0 < self.top_p <= 1.0:
            raise ValueError(f"top_p must be in (0, 1], got {self.top_p}.")
405
406
407
        # quietly accept -1 as disabled, but prefer 0
        if self.top_k < -1:
            raise ValueError(f"top_k must be 0 (disable), or at least 1, "
408
                             f"got {self.top_k}.")
409
410
411
        if not isinstance(self.top_k, int):
            raise TypeError(
                f"top_k must be an integer, got {type(self.top_k).__name__}")
Roy's avatar
Roy committed
412
413
414
        if not 0.0 <= self.min_p <= 1.0:
            raise ValueError("min_p must be in [0, 1], got "
                             f"{self.min_p}.")
415
        if self.max_tokens is not None and self.max_tokens < 1:
416
417
            raise ValueError(
                f"max_tokens must be at least 1, got {self.max_tokens}.")
418
419
420
421
422
423
424
        if self.min_tokens < 0:
            raise ValueError(f"min_tokens must be greater than or equal to 0, "
                             f"got {self.min_tokens}.")
        if self.max_tokens is not None and self.min_tokens > self.max_tokens:
            raise ValueError(
                f"min_tokens must be less than or equal to "
                f"max_tokens={self.max_tokens}, got {self.min_tokens}.")
425
426
        if (self.logprobs is not None and self.logprobs != -1
                and self.logprobs < 0):
427
            raise ValueError(
428
                f"logprobs must be non-negative or -1, got {self.logprobs}.")
429
430
431
432
433
        if (self.prompt_logprobs is not None and self.prompt_logprobs != -1
                and self.prompt_logprobs < 0):
            raise ValueError(
                f"prompt_logprobs must be non-negative or -1, got "
                f"{self.prompt_logprobs}.")
434
        if (self.truncate_prompt_tokens is not None
435
436
437
438
439
                and (self.truncate_prompt_tokens == 0
                     or self.truncate_prompt_tokens < -1)):
            raise ValueError(
                f"truncate_prompt_tokens must be an integer >= 1 or -1, "
                f"got {self.truncate_prompt_tokens}")
440
441
442
443
        assert isinstance(self.stop_token_ids, list)
        if not all(isinstance(st_id, int) for st_id in self.stop_token_ids):
            raise ValueError(f"stop_token_ids must contain only integers, "
                             f"got {self.stop_token_ids}.")
444
        assert isinstance(self.stop, list)
445
446
        if any(not stop_str for stop_str in self.stop):
            raise ValueError("stop cannot contain an empty string.")
447
448
449
450
        if self.stop and not self.detokenize:
            raise ValueError(
                "stop strings are only supported when detokenize is True. "
                "Set detokenize=True to use stop.")
451
452
453
        if self.best_of != self._real_n and self.output_kind == (
                RequestOutputKind.DELTA):
            raise ValueError("best_of must equal n to use output_kind=DELTA")
454
455

    def _verify_greedy_sampling(self) -> None:
456
457
458
        if self.n > 1:
            raise ValueError("n must be 1 when using greedy sampling, "
                             f"got {self.n}.")
459

460
    def update_from_generation_config(
461
            self,
462
            generation_config: dict[str, Any],
463
            model_eos_token_id: Optional[int] = None) -> None:
464
        """Update if there are non-default values from generation_config"""
465
466
467
468

        if model_eos_token_id is not None:
            # Add the eos token id into the sampling_params to support
            # min_tokens processing.
469
            self._all_stop_token_ids.add(model_eos_token_id)
470

471
        # Update eos_token_id for generation
472
        if (eos_ids := generation_config.get("eos_token_id")) is not None:
473
            # it can be either int or list of int
474
475
476
477
478
479
480
            eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids)
            if model_eos_token_id is not None:
                # We don't need to include the primary eos_token_id in
                # stop_token_ids since it's handled separately for stopping
                # purposes.
                eos_ids.discard(model_eos_token_id)
            if eos_ids:
481
                self._all_stop_token_ids.update(eos_ids)
482
483
484
                if not self.ignore_eos:
                    eos_ids.update(self.stop_token_ids)
                    self.stop_token_ids = list(eos_ids)
485

486
    def update_from_tokenizer(self, tokenizer: AnyTokenizer) -> None:
487
        if not self.bad_words:
488
            return
489
        self._bad_words_token_ids = []
490
491
492
493
494
495
496
        for bad_word in self.bad_words:
            # To prohibit words both at the beginning
            # and in the middle of text
            # (related to add_prefix_space tokenizer parameter)
            for add_prefix_space in [False, True]:
                prefix = " " if add_prefix_space else ""
                prompt = prefix + bad_word.lstrip()
497
498
                prompt_token_ids = tokenizer.encode(text=prompt,
                                                    add_special_tokens=False)
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521

                # If no space at the beginning
                # or if prefix space produces a new word token
                if (not add_prefix_space) or (
                        add_prefix_space and prompt_token_ids[0]
                        != self._bad_words_token_ids[-1][0]
                        and len(prompt_token_ids) == len(
                            self._bad_words_token_ids[-1])):
                    self._bad_words_token_ids.append(prompt_token_ids)

        invalid_token_ids = [
            token_id for bad_words_token_ids in self._bad_words_token_ids
            for token_id in bad_words_token_ids
            if token_id < 0 or token_id > tokenizer.max_token_id
        ]
        if len(invalid_token_ids) > 0:
            raise ValueError(
                f"The model vocabulary size is {tokenizer.max_token_id+1},"
                f" but the following tokens"
                f" were specified as bad: {invalid_token_ids}."
                f" All token id values should be integers satisfying:"
                f" 0 <= token_id <= {tokenizer.max_token_id}.")

522
523
524
525
    @cached_property
    def sampling_type(self) -> SamplingType:
        if self.temperature < _SAMPLING_EPS:
            return SamplingType.GREEDY
Nick Hill's avatar
Nick Hill committed
526
527
        if self.seed is not None:
            return SamplingType.RANDOM_SEED
528
529
        return SamplingType.RANDOM

530
    @property
531
    def all_stop_token_ids(self) -> set[int]:
532
533
        return self._all_stop_token_ids

534
    @property
535
    def bad_words_token_ids(self) -> Optional[list[list[int]]]:
536
537
538
        # For internal use only. Backward compatibility not guaranteed
        return self._bad_words_token_ids

539
    def clone(self) -> "SamplingParams":
540
        """Deep copy, but maybe not the LogitsProcessor objects.
541

542
543
544
        LogitsProcessor objects may contain an arbitrary, nontrivial amount of
        data that is expensive to copy. However, if not copied, the processor
        needs to support parallel decoding for multiple sequences
545
546
547
548
        See https://github.com/vllm-project/vllm/issues/3087
        """

        logit_processor_refs = None if self.logits_processors is None else {
549
            id(lp): lp.clone() if hasattr(lp, 'clone') else lp
550
551
552
553
            for lp in self.logits_processors
        }
        return copy.deepcopy(self, memo=logit_processor_refs)

554
    def __repr__(self) -> str:
555
556
557
558
559
560
561
562
563
        return (
            f"SamplingParams(n={self.n}, "
            f"presence_penalty={self.presence_penalty}, "
            f"frequency_penalty={self.frequency_penalty}, "
            f"repetition_penalty={self.repetition_penalty}, "
            f"temperature={self.temperature}, "
            f"top_p={self.top_p}, "
            f"top_k={self.top_k}, "
            f"min_p={self.min_p}, "
Nick Hill's avatar
Nick Hill committed
564
            f"seed={self.seed}, "
565
566
            f"stop={self.stop}, "
            f"stop_token_ids={self.stop_token_ids}, "
567
            f"bad_words={self.bad_words}, "
568
569
570
            f"include_stop_str_in_output={self.include_stop_str_in_output}, "
            f"ignore_eos={self.ignore_eos}, "
            f"max_tokens={self.max_tokens}, "
571
            f"min_tokens={self.min_tokens}, "
572
573
574
575
            f"logprobs={self.logprobs}, "
            f"prompt_logprobs={self.prompt_logprobs}, "
            f"skip_special_tokens={self.skip_special_tokens}, "
            "spaces_between_special_tokens="
576
            f"{self.spaces_between_special_tokens}, "
577
            f"truncate_prompt_tokens={self.truncate_prompt_tokens}, "
578
            f"structured_outputs={self.structured_outputs}, "
579
            f"extra_args={self.extra_args})")
580
581
582
583
584
585
586
587
588
589
590
591


class BeamSearchParams(
        msgspec.Struct,
        omit_defaults=True,  # type: ignore[call-arg]
        # required for @cached_property.
        dict=True):  # type: ignore[call-arg]
    """Beam search parameters for text generation."""
    beam_width: int
    max_tokens: int
    ignore_eos: bool = False
    temperature: float = 0.0
592
    length_penalty: float = 1.0
593
    include_stop_str_in_output: bool = False