Unverified Commit bc8ed3c4 authored by Woosuk Kwon's avatar Woosuk Kwon Committed by GitHub
Browse files

[V1][Spec Decode] Use better defaults for N-gram (#15358)


Signed-off-by: default avatarWoosuk Kwon <woosuk.kwon@berkeley.edu>
parent b9bd76ca
...@@ -2012,18 +2012,30 @@ class SpeculativeConfig: ...@@ -2012,18 +2012,30 @@ class SpeculativeConfig:
if self.method in ("ngram", "[ngram]"): if self.method in ("ngram", "[ngram]"):
# Unified to "ngram" internally # Unified to "ngram" internally
self.method = "ngram" self.method = "ngram"
if self.prompt_lookup_min is None: # Set default values if not provided
self.prompt_lookup_min = 1 if (self.prompt_lookup_min is None
if self.prompt_lookup_max is None or self.prompt_lookup_max < 1: and self.prompt_lookup_max is None):
raise ValueError("prompt_lookup_max=" # TODO(woosuk): Tune these values. They are arbitrarily chosen.
f"{self.prompt_lookup_max} must be > 0") self.prompt_lookup_min = 5
self.prompt_lookup_max = 5
elif self.prompt_lookup_min is None:
assert self.prompt_lookup_max is not None
self.prompt_lookup_min = self.prompt_lookup_max
elif self.prompt_lookup_max is None:
assert self.prompt_lookup_min is not None
self.prompt_lookup_max = self.prompt_lookup_min
# Validate values
if self.prompt_lookup_min < 1: if self.prompt_lookup_min < 1:
raise ValueError("prompt_lookup_min=" raise ValueError(
f"{self.prompt_lookup_min} must be > 0") f"prompt_lookup_min={self.prompt_lookup_min} must be > 0")
if self.prompt_lookup_max < 1:
raise ValueError(
f"prompt_lookup_max={self.prompt_lookup_max} must be > 0")
if self.prompt_lookup_min > self.prompt_lookup_max: if self.prompt_lookup_min > self.prompt_lookup_max:
raise ValueError(f"prompt_lookup_min={self.prompt_lookup_min} " raise ValueError(
"cannot be larger than prompt_lookup_max=" f"prompt_lookup_min={self.prompt_lookup_min} must "
f"{self.prompt_lookup_max}") f"be <= prompt_lookup_max={self.prompt_lookup_max}")
# TODO: current we still need extract vocab_size from target model # TODO: current we still need extract vocab_size from target model
# config, in future, we may try refactor it out, and set # config, in future, we may try refactor it out, and set
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment