Unverified Commit bc8ed3c4 authored by Woosuk Kwon's avatar Woosuk Kwon Committed by GitHub
Browse files

[V1][Spec Decode] Use better defaults for N-gram (#15358)


Signed-off-by: default avatarWoosuk Kwon <woosuk.kwon@berkeley.edu>
parent b9bd76ca
......@@ -2012,18 +2012,30 @@ class SpeculativeConfig:
if self.method in ("ngram", "[ngram]"):
# Unified to "ngram" internally
self.method = "ngram"
if self.prompt_lookup_min is None:
self.prompt_lookup_min = 1
if self.prompt_lookup_max is None or self.prompt_lookup_max < 1:
raise ValueError("prompt_lookup_max="
f"{self.prompt_lookup_max} must be > 0")
# Set default values if not provided
if (self.prompt_lookup_min is None
and self.prompt_lookup_max is None):
# TODO(woosuk): Tune these values. They are arbitrarily chosen.
self.prompt_lookup_min = 5
self.prompt_lookup_max = 5
elif self.prompt_lookup_min is None:
assert self.prompt_lookup_max is not None
self.prompt_lookup_min = self.prompt_lookup_max
elif self.prompt_lookup_max is None:
assert self.prompt_lookup_min is not None
self.prompt_lookup_max = self.prompt_lookup_min
# Validate values
if self.prompt_lookup_min < 1:
raise ValueError("prompt_lookup_min="
f"{self.prompt_lookup_min} must be > 0")
raise ValueError(
f"prompt_lookup_min={self.prompt_lookup_min} must be > 0")
if self.prompt_lookup_max < 1:
raise ValueError(
f"prompt_lookup_max={self.prompt_lookup_max} must be > 0")
if self.prompt_lookup_min > self.prompt_lookup_max:
raise ValueError(f"prompt_lookup_min={self.prompt_lookup_min} "
"cannot be larger than prompt_lookup_max="
f"{self.prompt_lookup_max}")
raise ValueError(
f"prompt_lookup_min={self.prompt_lookup_min} must "
f"be <= prompt_lookup_max={self.prompt_lookup_max}")
# TODO: current we still need extract vocab_size from target model
# config, in future, we may try refactor it out, and set
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment