sampling_params.py 26.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
"""Sampling parameters for text generation."""
4

5
import copy
6
import warnings
7
from dataclasses import field
8
from enum import Enum, IntEnum
9
from functools import cached_property
10
from typing import Annotated, Any
11

12
import msgspec
13
from pydantic.dataclasses import dataclass
Woosuk Kwon's avatar
Woosuk Kwon committed
14

15
from vllm.logger import init_logger
16
from vllm.logits_process import LogitsProcessor
17
from vllm.transformers_utils.tokenizer import AnyTokenizer
18
from vllm.v1.serial_utils import PydanticMsgspecMixin
19
20
21

logger = init_logger(__name__)

22
_SAMPLING_EPS = 1e-5
23
_MAX_TEMP = 1e-2
Woosuk Kwon's avatar
Woosuk Kwon committed
24

25

26
27
28
class SamplingType(IntEnum):
    GREEDY = 0
    RANDOM = 1
Nick Hill's avatar
Nick Hill committed
29
    RANDOM_SEED = 2
30
31


32
33
# maybe make msgspec?
@dataclass
34
35
class StructuredOutputsParams:
    # One of these fields will be used to build a logit processor.
36
37
38
39
40
    json: str | dict | None = None
    regex: str | None = None
    choice: list[str] | None = None
    grammar: str | None = None
    json_object: bool | None = None
41
    # These are other options that can be set.
42
43
44
    disable_fallback: bool = False
    disable_any_whitespace: bool = False
    disable_additional_properties: bool = False
45
46
    whitespace_pattern: str | None = None
    structural_tag: str | None = None
47

48
    _backend: str | None = field(default=None, init=False)
49
50
51
    """CAUTION: Should only be set by Processor._validate_structured_output"""
    _backend_was_auto: bool = field(default=False, init=False)
    """CAUTION: Should only be set by Processor._validate_structured_output"""
52
53
54

    def __post_init__(self):
        """Validate that some fields are mutually exclusive."""
55
56
57
58
59
60
61
        count = sum(
            [
                self.json is not None,
                self.regex is not None,
                self.choice is not None,
                self.grammar is not None,
                self.json_object is not None,
62
                self.structural_tag is not None,
63
64
            ]
        )
65
        if count > 1:
66
            raise ValueError(
67
                "You can only use one kind of structured outputs constraint "
68
69
                f"but multiple are specified: {self.__dict__}"
            )
70

71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
    def all_constraints_none(self) -> bool:
        """
        Returns True if all structured-output constraint fields are None.
        """
        return all(
            getattr(self, field) is None
            for field in (
                "json",
                "regex",
                "choice",
                "grammar",
                "json_object",
                "structural_tag",
            )
        )

    def all_non_structural_tag_constraints_none(self) -> bool:
        """
        Returns True if all structured-output constraint fields are None.
        """
        return all(
            getattr(self, field) is None
            for field in (
                "json",
                "regex",
                "choice",
                "grammar",
                "json_object",
            )
        )

102

103
104
105
106
107
108
109
110
@dataclass
class GuidedDecodingParams(StructuredOutputsParams):
    def __post_init__(self):
        warnings.warn(
            "GuidedDecodingParams is deprecated. This will be removed in "
            "v0.12.0 or v1.0.0, which ever is soonest. Please use "
            "StructuredOutputsParams instead.",
            DeprecationWarning,
111
112
            stacklevel=2,
        )
113
114
115
        return super().__post_init__()


116
117
118
119
120
class RequestOutputKind(Enum):
    # Return entire output so far in every RequestOutput
    CUMULATIVE = 0
    # Return only deltas in each RequestOutput
    DELTA = 1
121
    # Do not return intermediate RequestOutput
122
123
124
    FINAL_ONLY = 2


125
class SamplingParams(
126
    PydanticMsgspecMixin,
127
128
129
130
131
    msgspec.Struct,
    omit_defaults=True,  # type: ignore[call-arg]
    # required for @cached_property.
    dict=True,
):  # type: ignore[call-arg]
132
133
134
135
136
137
    """Sampling parameters for text generation.

    Overall, we follow the sampling parameters from the OpenAI text completion
    API (https://platform.openai.com/docs/api-reference/completions/create).
    In addition, we support beam search, which is not supported by OpenAI.
    """
Woosuk Kwon's avatar
Woosuk Kwon committed
138

139
    n: int = 1
140
141
142
143
144
145
146
    """Number of outputs to return for the given prompt request.

    NOTE:
        `AsyncLLM` streams outputs by default. When `n > 1`, all `n` outputs
        are generated and streamed cumulatively per request. To see all `n`
        outputs upon completion, use `output_kind=RequestOutputKind.FINAL_ONLY`
        in `SamplingParams`."""
147
    best_of: int | None = None
148
149
150
151
    """Number of output sequences that are generated from the prompt. From
    these `best_of` sequences, the top `n` sequences are returned. `best_of`
    must be greater than or equal to `n`. By default, `best_of` is set to `n`.
    Warning, this is only supported in V0."""
152
    _real_n: int | None = None
153
    presence_penalty: float = 0.0
154
155
156
    """Penalizes new tokens based on whether they appear in the generated text
    so far. Values > 0 encourage the model to use new tokens, while values < 0
    encourage the model to repeat tokens."""
157
    frequency_penalty: float = 0.0
158
159
160
    """Penalizes new tokens based on their frequency in the generated text so
    far. Values > 0 encourage the model to use new tokens, while values < 0
    encourage the model to repeat tokens."""
161
    repetition_penalty: float = 1.0
162
163
164
    """Penalizes new tokens based on whether they appear in the prompt and the
    generated text so far. Values > 1 encourage the model to use new tokens,
    while values < 1 encourage the model to repeat tokens."""
165
    temperature: float = 1.0
166
167
168
    """Controls the randomness of the sampling. Lower values make the model
    more deterministic, while higher values make the model more random. Zero
    means greedy sampling."""
169
    top_p: float = 1.0
170
171
    """Controls the cumulative probability of the top tokens to consider. Must
    be in (0, 1]. Set to 1 to consider all tokens."""
172
    top_k: int = 0
173
174
    """Controls the number of top tokens to consider. Set to 0 (or -1) to
    consider all tokens."""
175
    min_p: float = 0.0
176
177
178
    """Represents the minimum probability for a token to be considered,
    relative to the probability of the most likely token. Must be in [0, 1].
    Set to 0 to disable this."""
179
    seed: int | None = None
180
    """Random seed to use for the generation."""
181
    stop: str | list[str] | None = None
182
183
    """String(s) that stop the generation when they are generated. The returned
    output will not contain the stop strings."""
184
    stop_token_ids: list[int] | None = None
185
186
187
    """Token IDs that stop the generation when they are generated. The returned
    output will contain the stop tokens unless the stop tokens are special
    tokens."""
188
    ignore_eos: bool = False
189
190
    """Whether to ignore the EOS token and continue generating
    tokens after the EOS token is generated."""
191
    max_tokens: int | None = 16
192
    """Maximum number of tokens to generate per output sequence."""
193
    min_tokens: int = 0
194
195
    """Minimum number of tokens to generate per output sequence before EOS or
    `stop_token_ids` can be generated"""
196
    logprobs: int | None = None
197
198
199
200
201
202
203
    """Number of log probabilities to return per output token. When set to
    `None`, no probability is returned. If set to a non-`None` value, the
    result includes the log probabilities of the specified number of most
    likely tokens, as well as the chosen tokens. Note that the implementation
    follows the OpenAI API: The API will always return the log probability of
    the sampled token, so there may be up to `logprobs+1` elements in the
    response. When set to -1, return all `vocab_size` log probabilities."""
204
    prompt_logprobs: int | None = None
205
206
    """Number of log probabilities to return per prompt token.
    When set to -1, return all `vocab_size` log probabilities."""
207
208
209
210
    # NOTE: This parameter is only exposed at the engine level for now.
    # It is not exposed in the OpenAI API server, as the OpenAI API does
    # not support returning only a list of token IDs.
    detokenize: bool = True
211
    """Whether to detokenize the output."""
212
    skip_special_tokens: bool = True
213
    """Whether to skip special tokens in the output."""
214
    spaces_between_special_tokens: bool = True
215
    """Whether to add spaces between special tokens in the output."""
216
217
218
    # `list[LogitsProcessor] | None` type. We use Any here because
    # `list[LogitsProcessor] | None` type is not supported by msgspec.
    logits_processors: Any | None = None
219
220
    """Functions that modify logits based on previously generated tokens, and
    optionally prompt tokens as a first argument."""
221
    include_stop_str_in_output: bool = False
222
    """Whether to include the stop strings in output text."""
223
    truncate_prompt_tokens: Annotated[int, msgspec.Meta(ge=-1)] | None = None
224
225
226
    """If set to -1, will use the truncation size supported by the model. If
    set to an integer k, will use only the last k tokens from the prompt
    (i.e., left truncation). If set to `None`, truncation is disabled."""
227
    output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE
228
229
230
231

    # The below fields are not supposed to be used as an input.
    # They are set in post_init.
    output_text_buffer_length: int = 0
232
    _all_stop_token_ids: set[int] = msgspec.field(default_factory=set)
233

234
    # Fields used to construct logits processors
235
    structured_outputs: StructuredOutputsParams | None = None
236
    """Parameters for configuring structured outputs."""
237
    guided_decoding: GuidedDecodingParams | None = None
238
    """Deprecated alias for structured_outputs."""
239
    logit_bias: dict[int, float] | None = None
240
241
    """If provided, the engine will construct a logits processor that applies
    these logit biases."""
242
    allowed_token_ids: list[int] | None = None
243
244
    """If provided, the engine will construct a logits processor which only
    retains scores for the given token ids."""
245
    extra_args: dict[str, Any] | None = None
246
247
248
    """Arbitrary additional args, that can be used by custom sampling
    implementations, plugins, etc. Not used by any in-tree sampling
    implementations."""
249

250
    # Fields used for bad words
251
    bad_words: list[str] | None = None
252
253
254
    """Words that are not allowed to be generated. More precisely, only the
    last token of a corresponding token sequence is not allowed when the next
    generated token can complete the sequence."""
255
    _bad_words_token_ids: list[list[int]] | None = None
256

257
258
    skip_reading_prefix_cache: bool = None

259
260
    @staticmethod
    def from_optional(
261
262
263
264
265
266
267
        n: int | None = 1,
        best_of: int | None = None,
        presence_penalty: float | None = 0.0,
        frequency_penalty: float | None = 0.0,
        repetition_penalty: float | None = 1.0,
        temperature: float | None = 1.0,
        top_p: float | None = 1.0,
268
        top_k: int = 0,
269
        min_p: float = 0.0,
270
271
272
273
        seed: int | None = None,
        stop: str | list[str] | None = None,
        stop_token_ids: list[int] | None = None,
        bad_words: list[str] | None = None,
274
275
        include_stop_str_in_output: bool = False,
        ignore_eos: bool = False,
276
        max_tokens: int | None = 16,
277
        min_tokens: int = 0,
278
279
        logprobs: int | None = None,
        prompt_logprobs: int | None = None,
280
281
282
        detokenize: bool = True,
        skip_special_tokens: bool = True,
        spaces_between_special_tokens: bool = True,
283
284
        logits_processors: list[LogitsProcessor] | None = None,
        truncate_prompt_tokens: Annotated[int, msgspec.Meta(ge=-1)] | None = None,
285
        output_kind: RequestOutputKind = RequestOutputKind.CUMULATIVE,
286
287
288
289
290
        structured_outputs: StructuredOutputsParams | None = None,
        guided_decoding: GuidedDecodingParams | None = None,
        logit_bias: dict[int, float] | dict[str, float] | None = None,
        allowed_token_ids: list[int] | None = None,
        extra_args: dict[str, Any] | None = None,
291
    ) -> "SamplingParams":
292
        if logit_bias is not None:
293
294
            # Convert token_id to integer
            # Clamp the bias between -100 and 100 per OpenAI API spec
295
            logit_bias = {
296
                int(token): min(100.0, max(-100.0, bias))
297
298
                for token, bias in logit_bias.items()
            }
299
300
301
302
303
304
        if guided_decoding is not None:
            warnings.warn(
                "guided_decoding is deprecated. This will be removed in "
                "v0.12.0 or v1.0.0, which ever is soonest. Please use "
                "structured_outputs instead.",
                DeprecationWarning,
305
306
                stacklevel=2,
            )
307
308
            structured_outputs = guided_decoding
            guided_decoding = None
309

310
311
        return SamplingParams(
            n=1 if n is None else n,
312
            best_of=best_of,
313
314
            presence_penalty=0.0 if presence_penalty is None else presence_penalty,
            frequency_penalty=0.0 if frequency_penalty is None else frequency_penalty,
315
            repetition_penalty=1.0
316
317
            if repetition_penalty is None
            else repetition_penalty,
318
319
320
321
322
323
324
            temperature=1.0 if temperature is None else temperature,
            top_p=1.0 if top_p is None else top_p,
            top_k=top_k,
            min_p=min_p,
            seed=seed,
            stop=stop,
            stop_token_ids=stop_token_ids,
325
            bad_words=bad_words,
326
327
328
329
330
331
332
333
334
335
336
            include_stop_str_in_output=include_stop_str_in_output,
            ignore_eos=ignore_eos,
            max_tokens=max_tokens,
            min_tokens=min_tokens,
            logprobs=logprobs,
            prompt_logprobs=prompt_logprobs,
            detokenize=detokenize,
            skip_special_tokens=skip_special_tokens,
            spaces_between_special_tokens=spaces_between_special_tokens,
            logits_processors=logits_processors,
            truncate_prompt_tokens=truncate_prompt_tokens,
337
            output_kind=output_kind,
338
            structured_outputs=structured_outputs,
339
340
            logit_bias=logit_bias,
            allowed_token_ids=allowed_token_ids,
341
            extra_args=extra_args,
342
343
        )

344
    def __post_init__(self) -> None:
345
346
347
348
        # how we deal with `best_of`:
        # if `best_of` is not set, we default to `n`;
        # if `best_of` is set, we set `n` to `best_of`,
        # and set `_real_n` to the original `n`.
349
350
351
352
353
354
        # when we return the result, we will check
        # if we need to return `n` or `_real_n` results
        if self.best_of:
            if self.best_of < self.n:
                raise ValueError(
                    f"best_of must be greater than or equal to n, "
355
356
                    f"got n={self.n} and best_of={self.best_of}."
                )
357
358
359
            if not self._real_n:
                self._real_n = self.n
                self.n = self.best_of
360

361
        if 0 < self.temperature < _MAX_TEMP:
362
363
364
            logger.warning(
                "temperature %s is less than %s, which may cause numerical "
                "errors nan or inf in tensors. We have maxed it out to %s.",
365
366
367
368
                self.temperature,
                _MAX_TEMP,
                _MAX_TEMP,
            )
369
            self.temperature = max(self.temperature, _MAX_TEMP)
370

371
        if self.seed == -1:
372
            self.seed = None
373

374
        if self.stop is None:
375
            self.stop = []
376
377
        elif isinstance(self.stop, str):
            self.stop = [self.stop]
378

379
        if self.stop_token_ids is None:
380
            self.stop_token_ids = []
381
382
383
384

        if self.bad_words is None:
            self.bad_words = []

385
386
387
388
389
        if self.logprobs is True:
            self.logprobs = 1

        if self.prompt_logprobs is True:
            self.prompt_logprobs = 1
390

391
392
        # Number of characters to hold back for stop string evaluation
        # until sequence is finished.
393
        if self.stop and not self.include_stop_str_in_output:
394
395
            self.output_text_buffer_length = max(len(s) for s in self.stop) - 1

396
        self._verify_args()
397
398
399
400

        if self.temperature < _SAMPLING_EPS:
            # Zero temperature means greedy sampling.
            self.top_p = 1.0
401
            self.top_k = 0
402
403
            self.min_p = 0.0
            self._verify_greedy_sampling()
404

405
        # eos_token_id is added to this by the engine
406
        self._all_stop_token_ids.update(self.stop_token_ids)
407

408
409
410
411
412
413
        if self.guided_decoding is not None:
            warnings.warn(
                "guided_decoding is deprecated. This will be removed in "
                "v0.12.0 or v1.0.0, which ever is soonest. Please use "
                "structured_outputs instead.",
                DeprecationWarning,
414
415
                stacklevel=2,
            )
416
417
418
            self.structured_outputs = self.guided_decoding
            self.guided_decoding = None

419
420
421
422
423
424
        if self.skip_reading_prefix_cache is None:
            # If prefix caching is enabled,
            # the output of prompt logprobs may less than n_prompt_tokens,
            # we need to skip reading cache at this request.
            self.skip_reading_prefix_cache = self.prompt_logprobs is not None

425
    def _verify_args(self) -> None:
426
        if not isinstance(self.n, int):
427
            raise ValueError(f"n must be an int, but is of type {type(self.n)}")
428
429
        if self.n < 1:
            raise ValueError(f"n must be at least 1, got {self.n}.")
430
431
432
        if self.best_of is not None:
            if not isinstance(self.best_of, int):
                raise ValueError(
433
434
                    f"best_of must be an integer, got {type(self.best_of)}"
                )
435
            if self.best_of < 1:
436
                raise ValueError(f"best_of must be at least 1, got {self.best_of}")
437
438
439
            if self.best_of < self.n:
                raise ValueError(
                    f"best_of must be greater than or equal to n, "
440
441
                    f"got n={self.n} and best_of={self.best_of}."
                )
442
        if not -2.0 <= self.presence_penalty <= 2.0:
443
444
445
            raise ValueError(
                f"presence_penalty must be in [-2, 2], got {self.presence_penalty}."
            )
446
        if not -2.0 <= self.frequency_penalty <= 2.0:
447
448
449
            raise ValueError(
                f"frequency_penalty must be in [-2, 2], got {self.frequency_penalty}."
            )
450
451
452
        if self.repetition_penalty <= 0.0:
            raise ValueError(
                "repetition_penalty must be greater than zero, got "
453
454
                f"{self.repetition_penalty}."
            )
455
456
        if self.temperature < 0.0:
            raise ValueError(
457
458
                f"temperature must be non-negative, got {self.temperature}."
            )
459
460
        if not 0.0 < self.top_p <= 1.0:
            raise ValueError(f"top_p must be in (0, 1], got {self.top_p}.")
461
462
        # quietly accept -1 as disabled, but prefer 0
        if self.top_k < -1:
463
464
465
            raise ValueError(
                f"top_k must be 0 (disable), or at least 1, got {self.top_k}."
            )
466
467
        if not isinstance(self.top_k, int):
            raise TypeError(
468
469
                f"top_k must be an integer, got {type(self.top_k).__name__}"
            )
Roy's avatar
Roy committed
470
        if not 0.0 <= self.min_p <= 1.0:
471
            raise ValueError(f"min_p must be in [0, 1], got {self.min_p}.")
472
        if self.max_tokens is not None and self.max_tokens < 1:
473
            raise ValueError(f"max_tokens must be at least 1, got {self.max_tokens}.")
474
        if self.min_tokens < 0:
475
476
477
            raise ValueError(
                f"min_tokens must be greater than or equal to 0, got {self.min_tokens}."
            )
478
479
480
        if self.max_tokens is not None and self.min_tokens > self.max_tokens:
            raise ValueError(
                f"min_tokens must be less than or equal to "
481
482
483
                f"max_tokens={self.max_tokens}, got {self.min_tokens}."
            )
        if self.logprobs is not None and self.logprobs != -1 and self.logprobs < 0:
484
            raise ValueError(
485
486
487
488
489
490
491
                f"logprobs must be non-negative or -1, got {self.logprobs}."
            )
        if (
            self.prompt_logprobs is not None
            and self.prompt_logprobs != -1
            and self.prompt_logprobs < 0
        ):
492
493
            raise ValueError(
                f"prompt_logprobs must be non-negative or -1, got "
494
495
496
497
498
                f"{self.prompt_logprobs}."
            )
        if self.truncate_prompt_tokens is not None and (
            self.truncate_prompt_tokens == 0 or self.truncate_prompt_tokens < -1
        ):
499
500
            raise ValueError(
                f"truncate_prompt_tokens must be an integer >= 1 or -1, "
501
502
                f"got {self.truncate_prompt_tokens}"
            )
503
504
        assert isinstance(self.stop_token_ids, list)
        if not all(isinstance(st_id, int) for st_id in self.stop_token_ids):
505
506
507
            raise ValueError(
                f"stop_token_ids must contain only integers, got {self.stop_token_ids}."
            )
508
        assert isinstance(self.stop, list)
509
510
        if any(not stop_str for stop_str in self.stop):
            raise ValueError("stop cannot contain an empty string.")
511
512
513
        if self.stop and not self.detokenize:
            raise ValueError(
                "stop strings are only supported when detokenize is True. "
514
515
                "Set detokenize=True to use stop."
            )
516
        if self.best_of != self._real_n and self.output_kind == (
517
518
            RequestOutputKind.DELTA
        ):
519
            raise ValueError("best_of must equal n to use output_kind=DELTA")
520
521

    def _verify_greedy_sampling(self) -> None:
522
        if self.n > 1:
523
            raise ValueError(f"n must be 1 when using greedy sampling, got {self.n}.")
524

525
    def update_from_generation_config(
526
527
        self,
        generation_config: dict[str, Any],
528
        model_eos_token_id: int | None = None,
529
    ) -> None:
530
        """Update if there are non-default values from generation_config"""
531
532
533
534

        if model_eos_token_id is not None:
            # Add the eos token id into the sampling_params to support
            # min_tokens processing.
535
            self._all_stop_token_ids.add(model_eos_token_id)
536

537
        # Update eos_token_id for generation
538
        if (eos_ids := generation_config.get("eos_token_id")) is not None:
539
            # it can be either int or list of int
540
541
542
543
544
545
546
            eos_ids = {eos_ids} if isinstance(eos_ids, int) else set(eos_ids)
            if model_eos_token_id is not None:
                # We don't need to include the primary eos_token_id in
                # stop_token_ids since it's handled separately for stopping
                # purposes.
                eos_ids.discard(model_eos_token_id)
            if eos_ids:
547
                self._all_stop_token_ids.update(eos_ids)
548
549
550
                if not self.ignore_eos:
                    eos_ids.update(self.stop_token_ids)
                    self.stop_token_ids = list(eos_ids)
551

552
    def update_from_tokenizer(self, tokenizer: AnyTokenizer) -> None:
553
        if not self.bad_words:
554
            return
555
        self._bad_words_token_ids = []
556
557
558
559
560
561
562
        for bad_word in self.bad_words:
            # To prohibit words both at the beginning
            # and in the middle of text
            # (related to add_prefix_space tokenizer parameter)
            for add_prefix_space in [False, True]:
                prefix = " " if add_prefix_space else ""
                prompt = prefix + bad_word.lstrip()
563
564
565
                prompt_token_ids = tokenizer.encode(
                    text=prompt, add_special_tokens=False
                )
566
567
568
569

                # If no space at the beginning
                # or if prefix space produces a new word token
                if (not add_prefix_space) or (
570
571
572
573
                    add_prefix_space
                    and prompt_token_ids[0] != self._bad_words_token_ids[-1][0]
                    and len(prompt_token_ids) == len(self._bad_words_token_ids[-1])
                ):
574
575
576
                    self._bad_words_token_ids.append(prompt_token_ids)

        invalid_token_ids = [
577
578
            token_id
            for bad_words_token_ids in self._bad_words_token_ids
579
580
581
582
583
            for token_id in bad_words_token_ids
            if token_id < 0 or token_id > tokenizer.max_token_id
        ]
        if len(invalid_token_ids) > 0:
            raise ValueError(
584
                f"The model vocabulary size is {tokenizer.max_token_id + 1},"
585
586
587
                f" but the following tokens"
                f" were specified as bad: {invalid_token_ids}."
                f" All token id values should be integers satisfying:"
588
589
                f" 0 <= token_id <= {tokenizer.max_token_id}."
            )
590

591
592
593
594
    @cached_property
    def sampling_type(self) -> SamplingType:
        if self.temperature < _SAMPLING_EPS:
            return SamplingType.GREEDY
Nick Hill's avatar
Nick Hill committed
595
596
        if self.seed is not None:
            return SamplingType.RANDOM_SEED
597
598
        return SamplingType.RANDOM

599
    @property
600
    def all_stop_token_ids(self) -> set[int]:
601
602
        return self._all_stop_token_ids

603
    @property
604
    def bad_words_token_ids(self) -> list[list[int]] | None:
605
606
607
        # For internal use only. Backward compatibility not guaranteed
        return self._bad_words_token_ids

608
    def clone(self) -> "SamplingParams":
609
        """Deep copy, but maybe not the LogitsProcessor objects.
610

611
612
613
        LogitsProcessor objects may contain an arbitrary, nontrivial amount of
        data that is expensive to copy. However, if not copied, the processor
        needs to support parallel decoding for multiple sequences
614
615
616
        See https://github.com/vllm-project/vllm/issues/3087
        """

617
618
619
620
621
622
623
624
        logit_processor_refs = (
            None
            if self.logits_processors is None
            else {
                id(lp): lp.clone() if hasattr(lp, "clone") else lp
                for lp in self.logits_processors
            }
        )
625
626
        return copy.deepcopy(self, memo=logit_processor_refs)

627
    def __repr__(self) -> str:
628
629
630
631
632
633
634
635
636
        return (
            f"SamplingParams(n={self.n}, "
            f"presence_penalty={self.presence_penalty}, "
            f"frequency_penalty={self.frequency_penalty}, "
            f"repetition_penalty={self.repetition_penalty}, "
            f"temperature={self.temperature}, "
            f"top_p={self.top_p}, "
            f"top_k={self.top_k}, "
            f"min_p={self.min_p}, "
Nick Hill's avatar
Nick Hill committed
637
            f"seed={self.seed}, "
638
639
            f"stop={self.stop}, "
            f"stop_token_ids={self.stop_token_ids}, "
640
            f"bad_words={self.bad_words}, "
641
642
643
            f"include_stop_str_in_output={self.include_stop_str_in_output}, "
            f"ignore_eos={self.ignore_eos}, "
            f"max_tokens={self.max_tokens}, "
644
            f"min_tokens={self.min_tokens}, "
645
646
647
648
            f"logprobs={self.logprobs}, "
            f"prompt_logprobs={self.prompt_logprobs}, "
            f"skip_special_tokens={self.skip_special_tokens}, "
            "spaces_between_special_tokens="
649
            f"{self.spaces_between_special_tokens}, "
650
            f"truncate_prompt_tokens={self.truncate_prompt_tokens}, "
651
            f"structured_outputs={self.structured_outputs}, "
652
653
            f"extra_args={self.extra_args})"
        )
654
655
656


class BeamSearchParams(
657
658
659
660
661
    msgspec.Struct,
    omit_defaults=True,  # type: ignore[call-arg]
    # required for @cached_property.
    dict=True,
):  # type: ignore[call-arg]
662
    """Beam search parameters for text generation."""
663

664
665
666
667
    beam_width: int
    max_tokens: int
    ignore_eos: bool = False
    temperature: float = 0.0
668
    length_penalty: float = 1.0
669
    include_stop_str_in_output: bool = False