Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
OpenDAS
vllm_cscc
Commits
bc8ed3c4
Unverified
Commit
bc8ed3c4
authored
Mar 23, 2025
by
Woosuk Kwon
Committed by
GitHub
Mar 23, 2025
Browse files
[V1][Spec Decode] Use better defaults for N-gram (#15358)
Signed-off-by:
Woosuk Kwon
<
woosuk.kwon@berkeley.edu
>
parent
b9bd76ca
Changes
1
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
22 additions
and
10 deletions
+22
-10
vllm/config.py
vllm/config.py
+22
-10
No files found.
vllm/config.py
View file @
bc8ed3c4
...
@@ -2012,18 +2012,30 @@ class SpeculativeConfig:
...
@@ -2012,18 +2012,30 @@ class SpeculativeConfig:
if
self
.
method
in
(
"ngram"
,
"[ngram]"
):
if
self
.
method
in
(
"ngram"
,
"[ngram]"
):
# Unified to "ngram" internally
# Unified to "ngram" internally
self
.
method
=
"ngram"
self
.
method
=
"ngram"
if
self
.
prompt_lookup_min
is
None
:
# Set default values if not provided
self
.
prompt_lookup_min
=
1
if
(
self
.
prompt_lookup_min
is
None
if
self
.
prompt_lookup_max
is
None
or
self
.
prompt_lookup_max
<
1
:
and
self
.
prompt_lookup_max
is
None
):
raise
ValueError
(
"prompt_lookup_max="
# TODO(woosuk): Tune these values. They are arbitrarily chosen.
f
"
{
self
.
prompt_lookup_max
}
must be > 0"
)
self
.
prompt_lookup_min
=
5
self
.
prompt_lookup_max
=
5
elif
self
.
prompt_lookup_min
is
None
:
assert
self
.
prompt_lookup_max
is
not
None
self
.
prompt_lookup_min
=
self
.
prompt_lookup_max
elif
self
.
prompt_lookup_max
is
None
:
assert
self
.
prompt_lookup_min
is
not
None
self
.
prompt_lookup_max
=
self
.
prompt_lookup_min
# Validate values
if
self
.
prompt_lookup_min
<
1
:
if
self
.
prompt_lookup_min
<
1
:
raise
ValueError
(
"prompt_lookup_min="
raise
ValueError
(
f
"
{
self
.
prompt_lookup_min
}
must be > 0"
)
f
"prompt_lookup_min=
{
self
.
prompt_lookup_min
}
must be > 0"
)
if
self
.
prompt_lookup_max
<
1
:
raise
ValueError
(
f
"prompt_lookup_max=
{
self
.
prompt_lookup_max
}
must be > 0"
)
if
self
.
prompt_lookup_min
>
self
.
prompt_lookup_max
:
if
self
.
prompt_lookup_min
>
self
.
prompt_lookup_max
:
raise
ValueError
(
f
"prompt_lookup_min=
{
self
.
prompt_lookup_min
}
"
raise
ValueError
(
"cannot be larger than
prompt_lookup_m
ax=
"
f
"prompt_lookup_min=
{
self
.
prompt_lookup_m
in
}
must
"
f
"
{
self
.
prompt_lookup_max
}
"
)
f
"be <= prompt_lookup_max=
{
self
.
prompt_lookup_max
}
"
)
# TODO: current we still need extract vocab_size from target model
# TODO: current we still need extract vocab_size from target model
# config, in future, we may try refactor it out, and set
# config, in future, we may try refactor it out, and set
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment