Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "d3387750f191f3bcf6607db95436147bbccfacb3"
Unverified Commit a6977dbd authored by Harry Mellor's avatar Harry Mellor Committed by GitHub
Browse files

Simplify (and fix) passing of guided decoding backend options (#17008) 


Signed-off-by: default avatarHarry Mellor <19981378+hmellor@users.noreply.github.com>
parent 2fa2a50b
Show whitespace changes
Inline Side-by-side
Showing with 309 additions and 217 deletions
examples/online_serving/openai_chat_completion_structured_outputs.py
View file @ a6977dbd
...@@ -112,8 +112,8 @@ def extra_backend_options_completion(client: OpenAI, model: str): ...@@ -112,8 +112,8 @@ def extra_backend_options_completion(client: OpenAI, model: str):
"alan.turing@enigma.com\n") "alan.turing@enigma.com\n")
try: try:
# The no-fallback option forces vLLM to use xgrammar, so when it fails # The guided_decoding_disable_fallback option forces vLLM to use
# you get a 400 with the reason why # xgrammar, so when it fails you get a 400 with the reason why
completion = client.chat.completions.create( completion = client.chat.completions.create(
model=model, model=model,
messages=[{ messages=[{
...@@ -123,7 +123,8 @@ def extra_backend_options_completion(client: OpenAI, model: str): ...@@ -123,7 +123,8 @@ def extra_backend_options_completion(client: OpenAI, model: str):
extra_body={ extra_body={
"guided_regex": r"\w+@\w+\.com\n", "guided_regex": r"\w+@\w+\.com\n",
"stop": ["\n"], "stop": ["\n"],
"guided_decoding_backend": "xgrammar:no-fallback" "guided_decoding_backend": "xgrammar",
"guided_decoding_disable_fallback": True,
}, },
) )
return completion.choices[0].message.content return completion.choices[0].message.content
......
tests/entrypoints/llm/test_guided_generate.py
View file @ a6977dbd
...@@ -16,10 +16,11 @@ from vllm.sampling_params import GuidedDecodingParams, SamplingParams ...@@ -16,10 +16,11 @@ from vllm.sampling_params import GuidedDecodingParams, SamplingParams
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct" MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
GUIDED_DECODING_BACKENDS = [ GUIDED_DECODING_BACKENDS = [
"outlines", # (backend, disable_any_whitespace),
"lm-format-enforcer", ("outlines", False),
"xgrammar:disable-any-whitespace", ("lm-format-enforcer", False),
"guidance:disable-any-whitespace", ("xgrammar", True),
("guidance", True),
] ]
...@@ -36,13 +37,17 @@ def llm(): ...@@ -36,13 +37,17 @@ def llm():
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
def test_guided_regex(sample_regex, llm, guided_decoding_backend: str): GUIDED_DECODING_BACKENDS)
sampling_params = SamplingParams(temperature=0.8, def test_guided_regex(sample_regex, llm, guided_decoding_backend: str,
disable_any_whitespace: bool):
sampling_params = SamplingParams(
temperature=0.8,
top_p=0.95, top_p=0.95,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
regex=sample_regex, regex=sample_regex,
backend=guided_decoding_backend)) backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace))
outputs = llm.generate(prompts=[ outputs = llm.generate(prompts=[
f"Give an example IPv4 address with this regex: {sample_regex}" f"Give an example IPv4 address with this regex: {sample_regex}"
] * 2, ] * 2,
...@@ -62,14 +67,18 @@ def test_guided_regex(sample_regex, llm, guided_decoding_backend: str): ...@@ -62,14 +67,18 @@ def test_guided_regex(sample_regex, llm, guided_decoding_backend: str):
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
GUIDED_DECODING_BACKENDS)
def test_guided_json_completion(sample_json_schema, llm, def test_guided_json_completion(sample_json_schema, llm,
guided_decoding_backend: str): guided_decoding_backend: str,
sampling_params = SamplingParams(temperature=1.0, disable_any_whitespace: bool):
sampling_params = SamplingParams(
temperature=1.0,
max_tokens=1000, max_tokens=1000,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
json=sample_json_schema, json=sample_json_schema,
backend=guided_decoding_backend)) backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace))
outputs = llm.generate(prompts=[ outputs = llm.generate(prompts=[
f"Give an example JSON for an employee profile " f"Give an example JSON for an employee profile "
f"that fits this schema: {sample_json_schema}" f"that fits this schema: {sample_json_schema}"
...@@ -92,14 +101,18 @@ def test_guided_json_completion(sample_json_schema, llm, ...@@ -92,14 +101,18 @@ def test_guided_json_completion(sample_json_schema, llm,
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
GUIDED_DECODING_BACKENDS)
def test_guided_complex_json_completion(sample_complex_json_schema, llm, def test_guided_complex_json_completion(sample_complex_json_schema, llm,
guided_decoding_backend: str): guided_decoding_backend: str,
sampling_params = SamplingParams(temperature=1.0, disable_any_whitespace: bool):
sampling_params = SamplingParams(
temperature=1.0,
max_tokens=1000, max_tokens=1000,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
json=sample_complex_json_schema, json=sample_complex_json_schema,
backend=guided_decoding_backend)) backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace))
outputs = llm.generate(prompts=[ outputs = llm.generate(prompts=[
f"Give an example JSON for an assignment grade " f"Give an example JSON for an assignment grade "
f"that fits this schema: {sample_complex_json_schema}" f"that fits this schema: {sample_complex_json_schema}"
...@@ -123,14 +136,18 @@ def test_guided_complex_json_completion(sample_complex_json_schema, llm, ...@@ -123,14 +136,18 @@ def test_guided_complex_json_completion(sample_complex_json_schema, llm,
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
GUIDED_DECODING_BACKENDS)
def test_guided_definition_json_completion(sample_definition_json_schema, llm, def test_guided_definition_json_completion(sample_definition_json_schema, llm,
guided_decoding_backend: str): guided_decoding_backend: str,
sampling_params = SamplingParams(temperature=1.0, disable_any_whitespace: bool):
sampling_params = SamplingParams(
temperature=1.0,
max_tokens=1000, max_tokens=1000,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
json=sample_definition_json_schema, json=sample_definition_json_schema,
backend=guided_decoding_backend)) backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace))
outputs = llm.generate(prompts=[ outputs = llm.generate(prompts=[
f"Give an example JSON for solving 8x + 7 = -23 " f"Give an example JSON for solving 8x + 7 = -23 "
f"that fits this schema: {sample_definition_json_schema}" f"that fits this schema: {sample_definition_json_schema}"
...@@ -154,14 +171,18 @@ def test_guided_definition_json_completion(sample_definition_json_schema, llm, ...@@ -154,14 +171,18 @@ def test_guided_definition_json_completion(sample_definition_json_schema, llm,
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
GUIDED_DECODING_BACKENDS)
def test_guided_enum_json_completion(sample_enum_json_schema, llm, def test_guided_enum_json_completion(sample_enum_json_schema, llm,
guided_decoding_backend: str): guided_decoding_backend: str,
sampling_params = SamplingParams(temperature=1.0, disable_any_whitespace: bool):
sampling_params = SamplingParams(
temperature=1.0,
max_tokens=1000, max_tokens=1000,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
json=sample_enum_json_schema, json=sample_enum_json_schema,
backend=guided_decoding_backend)) backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace))
outputs = llm.generate(prompts=[ outputs = llm.generate(prompts=[
"Create a bug report JSON that fits this schema: " "Create a bug report JSON that fits this schema: "
f"{sample_enum_json_schema}. Make it for a high priority critical bug." f"{sample_enum_json_schema}. Make it for a high priority critical bug."
...@@ -195,14 +216,18 @@ def test_guided_enum_json_completion(sample_enum_json_schema, llm, ...@@ -195,14 +216,18 @@ def test_guided_enum_json_completion(sample_enum_json_schema, llm,
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
GUIDED_DECODING_BACKENDS)
def test_guided_choice_completion(sample_guided_choice, llm, def test_guided_choice_completion(sample_guided_choice, llm,
guided_decoding_backend: str): guided_decoding_backend: str,
sampling_params = SamplingParams(temperature=0.8, disable_any_whitespace: bool):
sampling_params = SamplingParams(
temperature=0.8,
top_p=0.95, top_p=0.95,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
choice=sample_guided_choice, choice=sample_guided_choice,
backend=guided_decoding_backend)) backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace))
outputs = llm.generate( outputs = llm.generate(
prompts="The best language for type-safe systems programming is ", prompts="The best language for type-safe systems programming is ",
sampling_params=sampling_params, sampling_params=sampling_params,
...@@ -221,15 +246,19 @@ def test_guided_choice_completion(sample_guided_choice, llm, ...@@ -221,15 +246,19 @@ def test_guided_choice_completion(sample_guided_choice, llm,
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
GUIDED_DECODING_BACKENDS)
def test_guided_grammar(sample_sql_statements, llm, def test_guided_grammar(sample_sql_statements, llm,
guided_decoding_backend: str): guided_decoding_backend: str,
sampling_params = SamplingParams(temperature=0.8, disable_any_whitespace: bool):
sampling_params = SamplingParams(
temperature=0.8,
top_p=0.95, top_p=0.95,
max_tokens=1000, max_tokens=1000,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
grammar=sample_sql_statements, grammar=sample_sql_statements,
backend=guided_decoding_backend)) backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace))
outputs = llm.generate( outputs = llm.generate(
prompts=("Generate a sql state that select col_1 from " prompts=("Generate a sql state that select col_1 from "
"table_1 where it is equals to 1"), "table_1 where it is equals to 1"),
...@@ -300,7 +329,8 @@ def test_disable_guided_decoding_fallback(sample_regex, llm): ...@@ -300,7 +329,8 @@ def test_disable_guided_decoding_fallback(sample_regex, llm):
top_p=0.95, top_p=0.95,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
json=unsupported_json, json=unsupported_json,
backend="xgrammar:no-fallback")) backend="xgrammar",
disable_fallback=True))
with pytest.raises( with pytest.raises(
ValueError, ValueError,
...@@ -312,14 +342,18 @@ def test_disable_guided_decoding_fallback(sample_regex, llm): ...@@ -312,14 +342,18 @@ def test_disable_guided_decoding_fallback(sample_regex, llm):
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
def test_guided_json_object(llm, guided_decoding_backend: str): GUIDED_DECODING_BACKENDS)
sampling_params = SamplingParams(temperature=1.0, def test_guided_json_object(llm, guided_decoding_backend: str,
disable_any_whitespace: bool):
sampling_params = SamplingParams(
temperature=1.0,
max_tokens=100, max_tokens=100,
n=2, n=2,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
json_object=True, json_object=True,
backend=guided_decoding_backend)) backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace))
outputs = llm.generate( outputs = llm.generate(
prompts=("Generate a JSON object with curly braces for a person with " prompts=("Generate a JSON object with curly braces for a person with "
...@@ -337,7 +371,7 @@ def test_guided_json_object(llm, guided_decoding_backend: str): ...@@ -337,7 +371,7 @@ def test_guided_json_object(llm, guided_decoding_backend: str):
print(generated_text) print(generated_text)
assert generated_text is not None assert generated_text is not None
if 'disable-any-whitespace' in guided_decoding_backend: if disable_any_whitespace:
assert "\n" not in generated_text assert "\n" not in generated_text
# Parse to verify it is valid JSON # Parse to verify it is valid JSON
...@@ -359,14 +393,18 @@ class CarDescription(BaseModel): ...@@ -359,14 +393,18 @@ class CarDescription(BaseModel):
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str): GUIDED_DECODING_BACKENDS)
def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str,
disable_any_whitespace: bool):
json_schema = CarDescription.model_json_schema() json_schema = CarDescription.model_json_schema()
sampling_params = SamplingParams(temperature=1.0, sampling_params = SamplingParams(
temperature=1.0,
max_tokens=1000, max_tokens=1000,
guided_decoding=GuidedDecodingParams( guided_decoding=GuidedDecodingParams(
json=json_schema, json=json_schema,
backend=guided_decoding_backend)) backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace))
outputs = llm.generate( outputs = llm.generate(
prompts="Generate a JSON with the brand, model and car_type of" prompts="Generate a JSON with the brand, model and car_type of"
"the most iconic car from the 90's", "the most iconic car from the 90's",
...@@ -387,9 +425,10 @@ def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str): ...@@ -387,9 +425,10 @@ def test_guided_json_completion_with_enum(llm, guided_decoding_backend: str):
@pytest.mark.skip_global_cleanup @pytest.mark.skip_global_cleanup
@pytest.mark.parametrize("guided_decoding_backend", GUIDED_DECODING_BACKENDS) @pytest.mark.parametrize("guided_decoding_backend,disable_any_whitespace",
def test_guided_number_range_json_completion(llm, GUIDED_DECODING_BACKENDS)
guided_decoding_backend: str): def test_guided_number_range_json_completion(llm, guided_decoding_backend: str,
disable_any_whitespace: bool):
sample_output_schema = { sample_output_schema = {
"type": "object", "type": "object",
"properties": { "properties": {
...@@ -413,8 +452,10 @@ def test_guided_number_range_json_completion(llm, ...@@ -413,8 +452,10 @@ def test_guided_number_range_json_completion(llm,
sampling_params = SamplingParams( sampling_params = SamplingParams(
temperature=1.0, temperature=1.0,
max_tokens=1000, max_tokens=1000,
guided_decoding=GuidedDecodingParams(json=sample_output_schema, guided_decoding=GuidedDecodingParams(
backend=guided_decoding_backend), json=sample_output_schema,
backend=guided_decoding_backend,
disable_any_whitespace=disable_any_whitespace),
) )
outputs = llm.generate( outputs = llm.generate(
prompts=[ prompts=[
...@@ -466,8 +507,12 @@ def test_guidance_no_additional_properties(llm): ...@@ -466,8 +507,12 @@ def test_guidance_no_additional_properties(llm):
"large JSON object with key-value pairs a1=b1, a2=b2, ..., a20=b20" "large JSON object with key-value pairs a1=b1, a2=b2, ..., a20=b20"
"<|im_end|>\n<|im_start|>assistant\n") "<|im_end|>\n<|im_start|>assistant\n")
def generate_with_backend(backend): def generate_with_backend(backend, disable_additional_properties):
guided_params = GuidedDecodingParams(json=schema, backend=backend) guided_params = GuidedDecodingParams(
json=schema,
backend=backend,
disable_any_whitespace=True,
disable_additional_properties=disable_additional_properties)
sampling_params = SamplingParams(temperature=0, sampling_params = SamplingParams(temperature=0,
max_tokens=256, max_tokens=256,
guided_decoding=guided_params) guided_decoding=guided_params)
...@@ -481,7 +526,7 @@ def test_guidance_no_additional_properties(llm): ...@@ -481,7 +526,7 @@ def test_guidance_no_additional_properties(llm):
jsonschema.validate(instance=parsed_json, schema=schema) jsonschema.validate(instance=parsed_json, schema=schema)
return parsed_json return parsed_json
base_generated = generate_with_backend('guidance:disable-any-whitespace') base_generated = generate_with_backend("guidance", False)
assert "a1" in base_generated assert "a1" in base_generated
assert "a2" in base_generated assert "a2" in base_generated
assert "a3" in base_generated assert "a3" in base_generated
...@@ -490,8 +535,7 @@ def test_guidance_no_additional_properties(llm): ...@@ -490,8 +535,7 @@ def test_guidance_no_additional_properties(llm):
assert "a5" in base_generated assert "a5" in base_generated
assert "a6" in base_generated assert "a6" in base_generated
generated = generate_with_backend( generated = generate_with_backend("guidance", True)
'guidance:no-additional-properties,disable-any-whitespace')
assert "a1" in generated assert "a1" in generated
assert "a2" in generated assert "a2" in generated
assert "a3" in generated assert "a3" in generated
......
tests/model_executor/test_guided_processors.py
View file @ a6977dbd
...@@ -202,12 +202,15 @@ def test_multiple_guided_options_not_allowed(sample_json_schema, sample_regex): ...@@ -202,12 +202,15 @@ def test_multiple_guided_options_not_allowed(sample_json_schema, sample_regex):
def test_guided_decoding_backend_options(): def test_guided_decoding_backend_options():
"""Test backend-specific options""" """Test backend-specific options"""
params = GuidedDecodingParams( with pytest.warns(DeprecationWarning):
backend="xgrammar:option-1,option-2,option-3") guided_decoding_params = GuidedDecodingParams(
assert params.backend_options() == ["option-1", "option-2", "option-3"] backend=
"xgrammar:no-fallback,disable-any-whitespace,no-additional-properties"
no_fallback = GuidedDecodingParams(backend="xgrammar:option-1,no-fallback") )
assert no_fallback.no_fallback() assert guided_decoding_params.backend == "xgrammar"
assert guided_decoding_params.disable_fallback
assert guided_decoding_params.disable_any_whitespace
assert guided_decoding_params.disable_additional_properties
def test_pickle_xgrammar_tokenizer_data(): def test_pickle_xgrammar_tokenizer_data():
......
tests/v1/entrypoints/llm/test_struct_output_generate.py
View file @ a6977dbd
...@@ -17,15 +17,12 @@ from vllm.platforms import current_platform ...@@ -17,15 +17,12 @@ from vllm.platforms import current_platform
from vllm.sampling_params import GuidedDecodingParams, SamplingParams from vllm.sampling_params import GuidedDecodingParams, SamplingParams
PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [ PARAMS_MODELS_BACKENDS_TOKENIZER_MODE = [
("mistralai/Ministral-8B-Instruct-2410", "xgrammar:disable-any-whitespace", ("mistralai/Ministral-8B-Instruct-2410", "xgrammar", "auto"),
"auto"), ("mistralai/Ministral-8B-Instruct-2410", "guidance", "auto"),
("mistralai/Ministral-8B-Instruct-2410", "guidance:disable-any-whitespace", ("mistralai/Ministral-8B-Instruct-2410", "xgrammar", "mistral"),
"auto"), ("Qwen/Qwen2.5-1.5B-Instruct", "xgrammar", "auto"),
("mistralai/Ministral-8B-Instruct-2410", "xgrammar:disable-any-whitespace",
"mistral"),
("Qwen/Qwen2.5-1.5B-Instruct", "xgrammar:disable-any-whitespace", "auto"),
#FIXME: This test is flaky on CI thus disabled #FIXME: This test is flaky on CI thus disabled
#("Qwen/Qwen2.5-1.5B-Instruct", "guidance:disable-any-whitespace", "auto"), #("Qwen/Qwen2.5-1.5B-Instruct", "guidance", "auto"),
] ]
PARAMS_MODELS_TOKENIZER_MODE = [ PARAMS_MODELS_TOKENIZER_MODE = [
...@@ -73,6 +70,7 @@ def test_structured_output( ...@@ -73,6 +70,7 @@ def test_structured_output(
enforce_eager=enforce_eager, enforce_eager=enforce_eager,
max_model_len=1024, max_model_len=1024,
guided_decoding_backend=guided_decoding_backend, guided_decoding_backend=guided_decoding_backend,
guided_decoding_disable_any_whitespace=True,
tokenizer_mode=tokenizer_mode) tokenizer_mode=tokenizer_mode)
# #
...@@ -98,7 +96,6 @@ def test_structured_output( ...@@ -98,7 +96,6 @@ def test_structured_output(
generated_text = output.outputs[0].text generated_text = output.outputs[0].text
assert generated_text is not None assert generated_text is not None
if 'disable-any-whitespace' in guided_decoding_backend:
assert "\n" not in generated_text assert "\n" not in generated_text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}") print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
output_json = json.loads(generated_text) output_json = json.loads(generated_text)
...@@ -520,10 +517,11 @@ def test_structured_output_auto_mode( ...@@ -520,10 +517,11 @@ def test_structured_output_auto_mode(
def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch): def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_USE_V1", "1") monkeypatch.setenv("VLLM_USE_V1", "1")
backend = 'guidance:no-additional-properties,disable-any-whitespace'
llm = LLM(model="Qwen/Qwen2.5-1.5B-Instruct", llm = LLM(model="Qwen/Qwen2.5-1.5B-Instruct",
max_model_len=1024, max_model_len=1024,
guided_decoding_backend=backend) guided_decoding_backend="guidance",
guided_decoding_disable_any_whitespace=True,
guided_decoding_disable_additional_properties=True)
schema = { schema = {
'type': 'object', 'type': 'object',
...@@ -548,7 +546,11 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch): ...@@ -548,7 +546,11 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
"<|im_end|>\n<|im_start|>assistant\n") "<|im_end|>\n<|im_start|>assistant\n")
def generate_with_backend(backend): def generate_with_backend(backend):
guided_params = GuidedDecodingParams(json=schema, backend=backend) guided_params = GuidedDecodingParams(
json=schema,
backend=backend,
disable_any_whitespace=True,
disable_additional_properties=True)
sampling_params = SamplingParams(temperature=0, sampling_params = SamplingParams(temperature=0,
max_tokens=256, max_tokens=256,
guided_decoding=guided_params) guided_decoding=guided_params)
...@@ -562,8 +564,7 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch): ...@@ -562,8 +564,7 @@ def test_guidance_no_additional_properties(monkeypatch: pytest.MonkeyPatch):
jsonschema.validate(instance=parsed_json, schema=schema) jsonschema.validate(instance=parsed_json, schema=schema)
return parsed_json return parsed_json
generated = generate_with_backend( generated = generate_with_backend("guidance")
'guidance:no-additional-properties,disable-any-whitespace')
assert "a1" in generated assert "a1" in generated
assert "a2" in generated assert "a2" in generated
assert "a3" in generated assert "a3" in generated
......
tests/v1/test_oracle.py
View file @ a6977dbd
...@@ -57,7 +57,8 @@ def test_unsupported_configs(monkeypatch): ...@@ -57,7 +57,8 @@ def test_unsupported_configs(monkeypatch):
with pytest.raises(NotImplementedError): with pytest.raises(NotImplementedError):
AsyncEngineArgs( AsyncEngineArgs(
model=MODEL, model=MODEL,
guided_decoding_backend="lm-format-enforcer:no-fallback", guided_decoding_backend="lm-format-enforcer",
guided_decoding_disable_fallback=True,
).create_engine_config() ).create_engine_config()
with pytest.raises(NotImplementedError): with pytest.raises(NotImplementedError):
......
vllm/config.py
View file @ a6977dbd
...@@ -17,12 +17,14 @@ from dataclasses import (MISSING, dataclass, field, fields, is_dataclass, ...@@ -17,12 +17,14 @@ from dataclasses import (MISSING, dataclass, field, fields, is_dataclass,
from importlib.util import find_spec from importlib.util import find_spec
from pathlib import Path from pathlib import Path
from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Final, Literal, from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Final, Literal,
Optional, Protocol, TypeVar, Union, get_args, get_origin) Optional, Protocol, TypeVar, Union, cast, get_args,
get_origin)
import torch import torch
from pydantic import BaseModel, Field, PrivateAttr from pydantic import BaseModel, Field, PrivateAttr
from torch.distributed import ProcessGroup, ReduceOp from torch.distributed import ProcessGroup, ReduceOp
from transformers import PretrainedConfig from transformers import PretrainedConfig
from typing_extensions import deprecated
import vllm.envs as envs import vllm.envs as envs
from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass from vllm.compilation.inductor_pass import CallableInductorPass, InductorPass
...@@ -32,7 +34,6 @@ from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS, ...@@ -32,7 +34,6 @@ from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
get_quantization_config) get_quantization_config)
from vllm.model_executor.models import ModelRegistry from vllm.model_executor.models import ModelRegistry
from vllm.platforms import CpuArchEnum, current_platform from vllm.platforms import CpuArchEnum, current_platform
from vllm.sampling_params import GuidedDecodingParams
from vllm.tracing import is_otel_available, otel_import_error_traceback from vllm.tracing import is_otel_available, otel_import_error_traceback
from vllm.transformers_utils.config import ( from vllm.transformers_utils.config import (
ConfigFormat, get_config, get_hf_image_processor_config, ConfigFormat, get_config, get_hf_image_processor_config,
...@@ -344,7 +345,7 @@ class ModelConfig: ...@@ -344,7 +345,7 @@ class ModelConfig:
def __init__( def __init__(
self, self,
model: str, model: str,
task: Union[TaskOption, Literal["draft"]], task: Literal[TaskOption, Literal["draft"]],
tokenizer: str, tokenizer: str,
tokenizer_mode: str, tokenizer_mode: str,
trust_remote_code: bool, trust_remote_code: bool,
...@@ -701,7 +702,7 @@ class ModelConfig: ...@@ -701,7 +702,7 @@ class ModelConfig:
def _resolve_task( def _resolve_task(
self, self,
task_option: Union[TaskOption, Literal["draft"]], task_option: Literal[TaskOption, Literal["draft"]],
) -> tuple[set[_ResolvedTask], _ResolvedTask]: ) -> tuple[set[_ResolvedTask], _ResolvedTask]:
if task_option == "draft": if task_option == "draft":
return {"draft"}, "draft" return {"draft"}, "draft"
...@@ -3185,13 +3186,36 @@ GuidedDecodingBackend = Literal[GuidedDecodingBackendV0, ...@@ -3185,13 +3186,36 @@ GuidedDecodingBackend = Literal[GuidedDecodingBackendV0,
class DecodingConfig: class DecodingConfig:
"""Dataclass which contains the decoding strategy of the engine.""" """Dataclass which contains the decoding strategy of the engine."""
guided_decoding_backend: GuidedDecodingBackend = \ @property
"auto" if envs.VLLM_USE_V1 else "xgrammar" @deprecated(
"`guided_decoding_backend` is deprecated and has been renamed to "
"`backend`. This will be removed in v0.10.0. Please use the "
"`backend` argument instead.")
def guided_decoding_backend(self) -> GuidedDecodingBackend:
return self.backend
@guided_decoding_backend.setter
def guided_decoding_backend(self, value: GuidedDecodingBackend):
self.backend = value
backend: GuidedDecodingBackend = "auto" if envs.VLLM_USE_V1 else "xgrammar"
"""Which engine will be used for guided decoding (JSON schema / regex etc) """Which engine will be used for guided decoding (JSON schema / regex etc)
by default. With "auto", we will make opinionated choices based on request by default. With "auto", we will make opinionated choices based on request
contents and what the backend libraries currently support, so the behavior contents and what the backend libraries currently support, so the behavior
is subject to change in each release.""" is subject to change in each release."""
disable_fallback: bool = False
"""If `True`, vLLM will not fallback to a different backend on error."""
disable_any_whitespace: bool = False
"""If `True`, the model will not generate any whitespace during guided
decoding. This is only supported for xgrammar and guidance backends."""
disable_additional_properties: bool = False
"""If `True`, the `guidance` backend will not use `additionalProperties`
in the JSON schema. This is only supported for the `guidance` backend and
is used to better align its behaviour with `outlines` and `xgrammar`."""
reasoning_backend: Optional[str] = None reasoning_backend: Optional[str] = None
"""Select the reasoning parser depending on the model that you're using. """Select the reasoning parser depending on the model that you're using.
This is used to parse the reasoning content into OpenAI API format. This is used to parse the reasoning content into OpenAI API format.
...@@ -3217,15 +3241,41 @@ class DecodingConfig: ...@@ -3217,15 +3241,41 @@ class DecodingConfig:
return hash_str return hash_str
def __post_init__(self): def __post_init__(self):
backend = GuidedDecodingParams( if ":" in self.backend:
backend=self.guided_decoding_backend).backend_name self._extract_backend_options()
if envs.VLLM_USE_V1: if envs.VLLM_USE_V1:
valid_guided_backends = get_args(GuidedDecodingBackendV1) valid_guided_backends = get_args(GuidedDecodingBackendV1)
else: else:
valid_guided_backends = get_args(GuidedDecodingBackendV0) valid_guided_backends = get_args(GuidedDecodingBackendV0)
if backend not in valid_guided_backends: if self.backend not in valid_guided_backends:
raise ValueError(f"Invalid guided_decoding_backend '{backend}'," raise ValueError(f"Invalid backend '{self.backend}',"
f" must be one of {valid_guided_backends}") f" must be one of {valid_guided_backends}")
if (self.disable_any_whitespace
and self.backend not in ("xgrammar", "guidance")):
raise ValueError("disable_any_whitespace is only supported for "
"xgrammar and guidance backends.")
if (self.disable_additional_properties and self.backend != "guidance"):
raise ValueError("disable_additional_properties is only supported "
"for the guidance backend.")
@deprecated(
"Passing guided decoding backend options inside backend in the format "
"'backend:...' is deprecated. This will be removed in v0.10.0. Please "
"use the dedicated arguments '--disable-fallback', "
"'--disable-any-whitespace' and '--disable-additional-properties' "
"instead.")
def _extract_backend_options(self):
"""Extract backend options from the backend string."""
backend, options = self.backend.split(":")
self.backend = cast(GuidedDecodingBackend, backend)
options_set = set(options.strip().split(","))
if "no-fallback" in options_set:
self.disable_fallback = True
if "disable-any-whitespace" in options_set:
self.disable_any_whitespace = True
if "no-additional-properties" in options_set:
self.disable_additional_properties = True
@dataclass @dataclass
......
vllm/engine/arg_utils.py
View file @ a6977dbd
...@@ -18,9 +18,9 @@ from vllm import version ...@@ -18,9 +18,9 @@ from vllm import version
from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig, from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
ConfigFormat, ConfigType, DecodingConfig, Device, ConfigFormat, ConfigType, DecodingConfig, Device,
DeviceConfig, DistributedExecutorBackend, DeviceConfig, DistributedExecutorBackend,
GuidedDecodingBackendV1, HfOverrides, GuidedDecodingBackend, GuidedDecodingBackendV1,
KVTransferConfig, LoadConfig, LoadFormat, LoRAConfig, HfOverrides, KVTransferConfig, LoadConfig, LoadFormat,
ModelConfig, ModelImpl, MultiModalConfig, LoRAConfig, ModelConfig, ModelImpl, MultiModalConfig,
ObservabilityConfig, ParallelConfig, PoolerConfig, ObservabilityConfig, ParallelConfig, PoolerConfig,
PrefixCachingHashAlgo, PromptAdapterConfig, PrefixCachingHashAlgo, PromptAdapterConfig,
SchedulerConfig, SchedulerPolicy, SpeculativeConfig, SchedulerConfig, SchedulerPolicy, SpeculativeConfig,
...@@ -317,7 +317,12 @@ class EngineArgs: ...@@ -317,7 +317,12 @@ class EngineArgs:
bool] = SchedulerConfig.enable_chunked_prefill bool] = SchedulerConfig.enable_chunked_prefill
disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input disable_chunked_mm_input: bool = SchedulerConfig.disable_chunked_mm_input
guided_decoding_backend: str = DecodingConfig.guided_decoding_backend guided_decoding_backend: GuidedDecodingBackend = DecodingConfig.backend
guided_decoding_disable_fallback: bool = DecodingConfig.disable_fallback
guided_decoding_disable_any_whitespace: bool = \
DecodingConfig.disable_any_whitespace
guided_decoding_disable_additional_properties: bool = \
DecodingConfig.disable_additional_properties
logits_processor_pattern: Optional[str] = None logits_processor_pattern: Optional[str] = None
speculative_config: Optional[Dict[str, Any]] = None speculative_config: Optional[Dict[str, Any]] = None
...@@ -498,9 +503,17 @@ class EngineArgs: ...@@ -498,9 +503,17 @@ class EngineArgs:
title="DecodingConfig", title="DecodingConfig",
description=DecodingConfig.__doc__, description=DecodingConfig.__doc__,
) )
guided_decoding_group.add_argument("--guided-decoding-backend",
**guided_decoding_kwargs["backend"])
guided_decoding_group.add_argument( guided_decoding_group.add_argument(
'--guided-decoding-backend', "--guided-decoding-disable-fallback",
**guided_decoding_kwargs["guided_decoding_backend"]) **guided_decoding_kwargs["disable_fallback"])
guided_decoding_group.add_argument(
"--guided-decoding-disable-any-whitespace",
**guided_decoding_kwargs["disable_any_whitespace"])
guided_decoding_group.add_argument(
"--guided-decoding-disable-additional-properties",
**guided_decoding_kwargs["disable_additional_properties"])
guided_decoding_group.add_argument( guided_decoding_group.add_argument(
"--reasoning-parser", "--reasoning-parser",
# This choices is a special case because it's not static # This choices is a special case because it's not static
...@@ -1244,7 +1257,11 @@ class EngineArgs: ...@@ -1244,7 +1257,11 @@ class EngineArgs:
if self.enable_prompt_adapter else None if self.enable_prompt_adapter else None
decoding_config = DecodingConfig( decoding_config = DecodingConfig(
guided_decoding_backend=self.guided_decoding_backend, backend=self.guided_decoding_backend,
disable_fallback=self.guided_decoding_disable_fallback,
disable_any_whitespace=self.guided_decoding_disable_any_whitespace,
disable_additional_properties=\
self.guided_decoding_disable_additional_properties,
reasoning_backend=self.reasoning_parser reasoning_backend=self.reasoning_parser
if self.enable_reasoning else None, if self.enable_reasoning else None,
) )
...@@ -1335,9 +1352,8 @@ class EngineArgs: ...@@ -1335,9 +1352,8 @@ class EngineArgs:
recommend_to_remove=True) recommend_to_remove=True)
return False return False
# remove backend options when doing this check if self.guided_decoding_backend not in get_args(
if self.guided_decoding_backend.split(':')[0] \ GuidedDecodingBackendV1):
not in get_args(GuidedDecodingBackendV1):
_raise_or_fallback( _raise_or_fallback(
feature_name= feature_name=
f"--guided-decoding-backend={self.guided_decoding_backend}", f"--guided-decoding-backend={self.guided_decoding_backend}",
......
vllm/engine/llm_engine.py
View file @ a6977dbd
...@@ -2091,7 +2091,7 @@ class LLMEngine: ...@@ -2091,7 +2091,7 @@ class LLMEngine:
tokenizer = self.get_tokenizer(lora_request=lora_request) tokenizer = self.get_tokenizer(lora_request=lora_request)
guided_decoding.backend = guided_decoding.backend or \ guided_decoding.backend = guided_decoding.backend or \
self.decoding_config.guided_decoding_backend self.decoding_config.backend
if self.decoding_config.reasoning_backend is not None: if self.decoding_config.reasoning_backend is not None:
logger.debug("Building with reasoning backend %s", logger.debug("Building with reasoning backend %s",
......
vllm/engine/multiprocessing/client.py
View file @ a6977dbd
...@@ -615,9 +615,9 @@ class MQLLMEngineClient(EngineClient): ...@@ -615,9 +615,9 @@ class MQLLMEngineClient(EngineClient):
build_guided_decoding_logits_processor_async( build_guided_decoding_logits_processor_async(
sampling_params=params, sampling_params=params,
tokenizer=await self.get_tokenizer(lora_request), tokenizer=await self.get_tokenizer(lora_request),
default_guided_backend=(self.decoding_config.guided_decoding_backend default_guided_backend=(self.decoding_config.backend
if self.decoding_config if self.decoding_config
else DecodingConfig.guided_decoding_backend), else DecodingConfig.backend),
model_config=self.model_config, model_config=self.model_config,
reasoning_backend=self.decoding_config.reasoning_backend, reasoning_backend=self.decoding_config.reasoning_backend,
) )
......
vllm/model_executor/guided_decoding/__init__.py
View file @ a6977dbd
...@@ -26,8 +26,8 @@ def maybe_backend_fallback( ...@@ -26,8 +26,8 @@ def maybe_backend_fallback(
def fallback_or_error(guided_params: GuidedDecodingParams, message: str, def fallback_or_error(guided_params: GuidedDecodingParams, message: str,
fallback: str) -> None: fallback: str) -> None:
"""Change the backend to the specified fallback with a warning log, """Change the backend to the specified fallback with a warning log,
or raise a ValueError if the `no-fallback` option is specified.""" or raise a ValueError if the `disable_fallback` option is specified."""
if guided_params.no_fallback(): if guided_params.disable_fallback:
raise ValueError(message) raise ValueError(message)
logger.warning("%s Falling back to use %s instead.", message, fallback) logger.warning("%s Falling back to use %s instead.", message, fallback)
...@@ -40,7 +40,7 @@ def maybe_backend_fallback( ...@@ -40,7 +40,7 @@ def maybe_backend_fallback(
guided_params.backend = "xgrammar" guided_params.backend = "xgrammar"
# lm-format-enforce doesn't support grammar, fallback to xgrammar # lm-format-enforce doesn't support grammar, fallback to xgrammar
if guided_params.backend_name == "lm-format-enforcer": if guided_params.backend == "lm-format-enforcer":
if guided_params.grammar is not None: if guided_params.grammar is not None:
fallback_or_error( fallback_or_error(
guided_params, guided_params,
...@@ -55,7 +55,7 @@ def maybe_backend_fallback( ...@@ -55,7 +55,7 @@ def maybe_backend_fallback(
"lm-format-enforcer does not support advanced JSON schema " "lm-format-enforcer does not support advanced JSON schema "
"features like patterns or numeric ranges.", "outlines") "features like patterns or numeric ranges.", "outlines")
if guided_params.backend_name == "xgrammar": if guided_params.backend == "xgrammar":
from vllm.model_executor.guided_decoding.xgrammar_decoding import ( from vllm.model_executor.guided_decoding.xgrammar_decoding import (
xgr_installed) xgr_installed)
...@@ -87,7 +87,7 @@ def maybe_backend_fallback( ...@@ -87,7 +87,7 @@ def maybe_backend_fallback(
guided_params, guided_params,
"xgrammar module cannot be imported successfully.", "outlines") "xgrammar module cannot be imported successfully.", "outlines")
if (guided_params.backend_name == "outlines" if (guided_params.backend == "outlines"
and guided_params.json_object is not None): and guided_params.json_object is not None):
# outlines doesn't support json_object, fallback to guidance # outlines doesn't support json_object, fallback to guidance
fallback_or_error(guided_params, fallback_or_error(guided_params,
...@@ -111,7 +111,7 @@ async def get_guided_decoding_logits_processor( ...@@ -111,7 +111,7 @@ async def get_guided_decoding_logits_processor(
guided_params = maybe_backend_fallback(guided_params) guided_params = maybe_backend_fallback(guided_params)
# CFG grammar not supported by LMFE, so we use outlines instead # CFG grammar not supported by LMFE, so we use outlines instead
if guided_params.backend_name == 'outlines': if guided_params.backend == 'outlines':
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
get_outlines_guided_decoding_logits_processor) get_outlines_guided_decoding_logits_processor)
...@@ -122,12 +122,12 @@ async def get_guided_decoding_logits_processor( ...@@ -122,12 +122,12 @@ async def get_guided_decoding_logits_processor(
get_local_lm_format_enforcer_guided_decoding_logits_processor) get_local_lm_format_enforcer_guided_decoding_logits_processor)
return get_local_lm_format_enforcer_guided_decoding_logits_processor( return get_local_lm_format_enforcer_guided_decoding_logits_processor(
guided_params, tokenizer) guided_params, tokenizer)
if guided_params.backend_name == 'xgrammar': if guided_params.backend == 'xgrammar':
from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
get_local_xgrammar_guided_decoding_logits_processor) get_local_xgrammar_guided_decoding_logits_processor)
return get_local_xgrammar_guided_decoding_logits_processor( return get_local_xgrammar_guided_decoding_logits_processor(
guided_params, tokenizer, model_config, reasoner) guided_params, tokenizer, model_config, reasoner)
if guided_params.backend_name == 'guidance': if guided_params.backend == 'guidance':
from vllm.model_executor.guided_decoding.guidance_decoding import ( from vllm.model_executor.guided_decoding.guidance_decoding import (
get_local_guidance_guided_decoding_logits_processor) get_local_guidance_guided_decoding_logits_processor)
return get_local_guidance_guided_decoding_logits_processor( return get_local_guidance_guided_decoding_logits_processor(
...@@ -152,23 +152,23 @@ def get_local_guided_decoding_logits_processor( ...@@ -152,23 +152,23 @@ def get_local_guided_decoding_logits_processor(
reasoner = reasoner_class(tokenizer) reasoner = reasoner_class(tokenizer)
# CFG grammar not supported by LMFE, so we use outlines instead # CFG grammar not supported by LMFE, so we use outlines instead
if guided_params.backend_name == 'outlines': if guided_params.backend == 'outlines':
# NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193 # NOTE: lazy import outlines to avoid https://github.com/vllm-project/vllm/issues/4193
from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa from vllm.model_executor.guided_decoding.outlines_decoding import ( # noqa
get_local_outlines_guided_decoding_logits_processor) get_local_outlines_guided_decoding_logits_processor)
return get_local_outlines_guided_decoding_logits_processor( return get_local_outlines_guided_decoding_logits_processor(
guided_params, tokenizer, reasoner) guided_params, tokenizer, reasoner)
if guided_params.backend_name == 'lm-format-enforcer': if guided_params.backend == 'lm-format-enforcer':
from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa from vllm.model_executor.guided_decoding.lm_format_enforcer_decoding import ( # noqa
get_local_lm_format_enforcer_guided_decoding_logits_processor) get_local_lm_format_enforcer_guided_decoding_logits_processor)
return get_local_lm_format_enforcer_guided_decoding_logits_processor( return get_local_lm_format_enforcer_guided_decoding_logits_processor(
guided_params, tokenizer) guided_params, tokenizer)
if guided_params.backend_name == 'xgrammar': if guided_params.backend == 'xgrammar':
from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa from vllm.model_executor.guided_decoding.xgrammar_decoding import ( # noqa
get_local_xgrammar_guided_decoding_logits_processor) get_local_xgrammar_guided_decoding_logits_processor)
return get_local_xgrammar_guided_decoding_logits_processor( return get_local_xgrammar_guided_decoding_logits_processor(
guided_params, tokenizer, model_config, reasoner) guided_params, tokenizer, model_config, reasoner)
if guided_params.backend_name == 'guidance': if guided_params.backend == 'guidance':
from vllm.model_executor.guided_decoding.guidance_decoding import ( from vllm.model_executor.guided_decoding.guidance_decoding import (
get_local_guidance_guided_decoding_logits_processor) get_local_guidance_guided_decoding_logits_processor)
return get_local_guidance_guided_decoding_logits_processor( return get_local_guidance_guided_decoding_logits_processor(
......
vllm/model_executor/guided_decoding/guidance_decoding.py
View file @ a6977dbd
...@@ -21,13 +21,12 @@ def get_local_guidance_guided_decoding_logits_processor( ...@@ -21,13 +21,12 @@ def get_local_guidance_guided_decoding_logits_processor(
""" """
grm = "" grm = ""
any_whitespace = 'disable-any-whitespace' not in \ any_whitespace = not guided_params.disable_any_whitespace
guided_params.backend_options()
if (guide_json := guided_params.json) is not None: if (guide_json := guided_params.json) is not None:
# Optionally set additionalProperties to False at the top-level # Optionally set additionalProperties to False at the top-level
# By default, other backends do not allow additional top-level # By default, other backends do not allow additional top-level
# properties, so this makes guidance more similar to other backends # properties, so this makes guidance more similar to other backends
if 'no-additional-properties' in guided_params.backend_options(): if guided_params.disable_additional_properties:
if not isinstance(guide_json, str): if not isinstance(guide_json, str):
guide_json = json.dumps(guide_json) guide_json = json.dumps(guide_json)
guide_json = process_for_additional_properties(guide_json) guide_json = process_for_additional_properties(guide_json)
......
vllm/model_executor/guided_decoding/xgrammar_decoding.py
View file @ a6977dbd
...@@ -175,8 +175,7 @@ class GrammarConfig: ...@@ -175,8 +175,7 @@ class GrammarConfig:
else: else:
json_str = guided_params.json json_str = guided_params.json
any_whitespace = 'disable-any-whitespace' not in \ any_whitespace = not guided_params.disable_any_whitespace
guided_params.backend_options()
# Check and log if model with xgrammar and whitespace have history # Check and log if model with xgrammar and whitespace have history
# of runaway generation of whitespaces. # of runaway generation of whitespaces.
...@@ -191,11 +190,10 @@ class GrammarConfig: ...@@ -191,11 +190,10 @@ class GrammarConfig:
model_with_warn = 'Qwen' model_with_warn = 'Qwen'
if model_with_warn is not None and any_whitespace: if model_with_warn is not None and any_whitespace:
msg = (f"{model_with_warn} " logger.info_once(
f"model detected, consider set " "%s model detected, consider setting "
f"`guided_backend=xgrammar:disable-any-whitespace` " "`disable_any_whitespace` to prevent runaway generation "
f"to prevent runaway generation of whitespaces.") "of whitespaces.", model_with_warn)
logger.info_once(msg)
# Validate the schema and raise ValueError here if it is invalid. # Validate the schema and raise ValueError here if it is invalid.
# This is to avoid exceptions in model execution, which will crash # This is to avoid exceptions in model execution, which will crash
# the engine worker process. # the engine worker process.
......
vllm/sampling_params.py
View file @ a6977dbd
...@@ -8,6 +8,7 @@ from typing import Annotated, Any, Optional, Union ...@@ -8,6 +8,7 @@ from typing import Annotated, Any, Optional, Union
import msgspec import msgspec
from pydantic import BaseModel from pydantic import BaseModel
from typing_extensions import deprecated
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.logits_process import LogitsProcessor from vllm.logits_process import LogitsProcessor
...@@ -37,6 +38,10 @@ class GuidedDecodingParams: ...@@ -37,6 +38,10 @@ class GuidedDecodingParams:
json_object: Optional[bool] = None json_object: Optional[bool] = None
"""These are other options that can be set""" """These are other options that can be set"""
backend: Optional[str] = None backend: Optional[str] = None
backend_was_auto: bool = False
disable_fallback: bool = False
disable_any_whitespace: bool = False
disable_additional_properties: bool = False
whitespace_pattern: Optional[str] = None whitespace_pattern: Optional[str] = None
structural_tag: Optional[str] = None structural_tag: Optional[str] = None
...@@ -68,36 +73,6 @@ class GuidedDecodingParams: ...@@ -68,36 +73,6 @@ class GuidedDecodingParams:
structural_tag=structural_tag, structural_tag=structural_tag,
) )
@property
def backend_name(self) -> str:
"""Return the backend name without any options.
For example if the backend is "xgrammar:no-fallback", returns "xgrammar"
"""
return (self.backend or "").split(":")[0]
def backend_options(self) -> list[str]:
"""Return the backend options as a list of strings."""
if not self.backend or ":" not in self.backend:
return []
return self.backend.split(":")[1].split(",")
def add_option(self, opt_name: str) -> None:
"""Adds an option to the backend options."""
if not self.backend:
self.backend = f":{opt_name}"
elif ":" not in self.backend:
self.backend += f":{opt_name}"
else:
options = set(self.backend_options())
options.add(opt_name)
self.backend = f"{self.backend_name}:{','.join(sorted(options))}"
def no_fallback(self) -> bool:
"""Returns True if the "no-fallback" option is supplied for the guided
decoding backend"""
return "no-fallback" in self.backend_options()
def __post_init__(self): def __post_init__(self):
"""Validate that some fields are mutually exclusive.""" """Validate that some fields are mutually exclusive."""
guide_count = sum([ guide_count = sum([
...@@ -109,6 +84,27 @@ class GuidedDecodingParams: ...@@ -109,6 +84,27 @@ class GuidedDecodingParams:
"You can only use one kind of guided decoding but multiple are " "You can only use one kind of guided decoding but multiple are "
f"specified: {self.__dict__}") f"specified: {self.__dict__}")
if self.backend is not None and ":" in self.backend:
self._extract_backend_options()
@deprecated(
"Passing guided decoding backend options inside backend in the format "
"'backend:...' is deprecated. This will be removed in v0.10.0. Please "
"use the dedicated arguments '--disable-fallback', "
"'--disable-any-whitespace' and '--disable-additional-properties' "
"instead.")
def _extract_backend_options(self):
"""Extract backend options from the backend string."""
assert isinstance(self.backend, str)
self.backend, options = self.backend.split(":")
options_set = set(options.strip().split(","))
if "no-fallback" in options_set:
self.disable_fallback = True
if "disable-any-whitespace" in options_set:
self.disable_any_whitespace = True
if "no-additional-properties" in options_set:
self.disable_additional_properties = True
class RequestOutputKind(Enum): class RequestOutputKind(Enum):
# Return entire output so far in every RequestOutput # Return entire output so far in every RequestOutput
......
vllm/v1/engine/processor.py
View file @ a6977dbd
...@@ -144,7 +144,7 @@ class Processor: ...@@ -144,7 +144,7 @@ class Processor:
if not params.guided_decoding or not self.decoding_config: if not params.guided_decoding or not self.decoding_config:
return return
engine_level_backend = self.decoding_config.guided_decoding_backend engine_level_backend = self.decoding_config.backend
if params.guided_decoding.backend: if params.guided_decoding.backend:
# Request-level backend selection is not supported in V1. # Request-level backend selection is not supported in V1.
# The values may differ if `params` is reused and was set # The values may differ if `params` is reused and was set
...@@ -152,8 +152,8 @@ class Processor: ...@@ -152,8 +152,8 @@ class Processor:
# request. We remember that it was set as a result of `auto` # request. We remember that it was set as a result of `auto`
# using the `_auto` option set on the backend in the params. # using the `_auto` option set on the backend in the params.
if (params.guided_decoding.backend != engine_level_backend if (params.guided_decoding.backend != engine_level_backend
and not (engine_level_backend == "auto" and "_auto" and not (engine_level_backend == "auto"
in params.guided_decoding.backend_options())): and params.guided_decoding.backend_was_auto)):
raise ValueError( raise ValueError(
"Request-level structured output backend selection is no " "Request-level structured output backend selection is no "
"longer supported. The request specified " "longer supported. The request specified "
...@@ -189,7 +189,7 @@ class Processor: ...@@ -189,7 +189,7 @@ class Processor:
# are not supported in xgrammar. Fall back to guidance. # are not supported in xgrammar. Fall back to guidance.
params.guided_decoding.backend = "guidance" params.guided_decoding.backend = "guidance"
# Remember that this backend was set automatically # Remember that this backend was set automatically
params.guided_decoding.add_option("_auto") params.guided_decoding.backend_was_auto = True
def process_inputs( def process_inputs(
self, self,
......
vllm/v1/structured_output/__init__.py
View file @ a6977dbd
...@@ -45,17 +45,17 @@ class StructuredOutputManager: ...@@ -45,17 +45,17 @@ class StructuredOutputManager:
# NOTE: We only support a single backend. We do NOT support different # NOTE: We only support a single backend. We do NOT support different
# backends on a per-request basis in V1 (for now, anyway...). # backends on a per-request basis in V1 (for now, anyway...).
if self.backend is None: if self.backend is None:
backend_name = request.sampling_params.guided_decoding.backend_name backend = request.sampling_params.guided_decoding.backend
if backend_name == "xgrammar": if backend == "xgrammar":
from vllm.v1.structured_output.backend_xgrammar import ( from vllm.v1.structured_output.backend_xgrammar import (
XgrammarBackend) XgrammarBackend)
self.backend = XgrammarBackend(self.vllm_config) self.backend = XgrammarBackend(self.vllm_config)
elif backend_name == "guidance": elif backend == "guidance":
self.backend = GuidanceBackend(self.vllm_config) self.backend = GuidanceBackend(self.vllm_config)
else: else:
raise ValueError( raise ValueError(
f"Unsupported structured output backend: {backend_name}") f"Unsupported structured output backend: {backend}")
grammar = self.executor.submit(self._async_create_grammar, request) grammar = self.executor.submit(self._async_create_grammar, request)
request.structured_output_request.grammar = grammar # type: ignore[assignment] request.structured_output_request.grammar = grammar # type: ignore[assignment]
......
vllm/v1/structured_output/backend_guidance.py
View file @ a6977dbd
...@@ -10,7 +10,7 @@ import torch ...@@ -10,7 +10,7 @@ import torch
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.sampling_params import GuidedDecodingParams, SamplingParams from vllm.sampling_params import SamplingParams
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
from vllm.utils import LazyLoader from vllm.utils import LazyLoader
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend, from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
...@@ -65,19 +65,10 @@ class GuidanceBackend(StructuredOutputBackend): ...@@ -65,19 +65,10 @@ class GuidanceBackend(StructuredOutputBackend):
self.vllm_config = vllm_config self.vllm_config = vllm_config
self.vocab_size = vllm_config.model_config.get_vocab_size() self.vocab_size = vllm_config.model_config.get_vocab_size()
self.disable_any_whitespace = False self.disable_any_whitespace = \
self.no_additional_properties = False vllm_config.decoding_config.disable_any_whitespace
backend_options = GuidedDecodingParams( self.disable_additional_properties = \
backend=vllm_config.decoding_config.guided_decoding_backend vllm_config.decoding_config.disable_additional_properties
).backend_options()
for option in backend_options:
if option == "disable-any-whitespace":
self.disable_any_whitespace = True
elif option == "no-additional-properties":
self.no_additional_properties = True
else:
raise ValueError(
f"Unsupported option for the guidance backend: {option}")
tokenizer = tokenizer_group.get_lora_tokenizer(None) tokenizer = tokenizer_group.get_lora_tokenizer(None)
self.ll_tokenizer = llguidance_hf.from_tokenizer( self.ll_tokenizer = llguidance_hf.from_tokenizer(
...@@ -87,7 +78,7 @@ class GuidanceBackend(StructuredOutputBackend): ...@@ -87,7 +78,7 @@ class GuidanceBackend(StructuredOutputBackend):
grammar_spec: str) -> StructuredOutputGrammar: grammar_spec: str) -> StructuredOutputGrammar:
self.serialized_grammar = serialize_guidance_grammar( self.serialized_grammar = serialize_guidance_grammar(
request_type, grammar_spec, self.disable_any_whitespace, request_type, grammar_spec, self.disable_any_whitespace,
self.no_additional_properties) self.disable_additional_properties)
ll_matcher = llguidance.LLMatcher( ll_matcher = llguidance.LLMatcher(
self.ll_tokenizer, self.ll_tokenizer,
...@@ -171,11 +162,11 @@ def serialize_guidance_grammar( ...@@ -171,11 +162,11 @@ def serialize_guidance_grammar(
request_type: StructuredOutputOptions, request_type: StructuredOutputOptions,
grammar_spec: Union[str, dict[str, Any]], grammar_spec: Union[str, dict[str, Any]],
disable_any_whitespace: bool = False, disable_any_whitespace: bool = False,
no_additional_properties: bool = False, disable_additional_properties: bool = False,
) -> str: ) -> str:
def _process_schema(grammar_spec: Union[str, dict[str, Any]], ) -> str: def _process_schema(grammar_spec: Union[str, dict[str, Any]], ) -> str:
if no_additional_properties: if disable_additional_properties:
grammar_spec = process_for_additional_properties(grammar_spec) grammar_spec = process_for_additional_properties(grammar_spec)
return llguidance.LLMatcher.grammar_from_json_schema( return llguidance.LLMatcher.grammar_from_json_schema(
grammar_spec, grammar_spec,
......
vllm/v1/structured_output/backend_xgrammar.py
View file @ a6977dbd
...@@ -9,7 +9,7 @@ import torch ...@@ -9,7 +9,7 @@ import torch
import vllm.envs import vllm.envs
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.sampling_params import GuidedDecodingParams, SamplingParams from vllm.sampling_params import SamplingParams
from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs from vllm.transformers_utils.tokenizer_group import init_tokenizer_from_configs
from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
from vllm.utils import LazyLoader from vllm.utils import LazyLoader
...@@ -37,16 +37,8 @@ class XgrammarBackend(StructuredOutputBackend): ...@@ -37,16 +37,8 @@ class XgrammarBackend(StructuredOutputBackend):
scheduler_config=vllm_config.scheduler_config, scheduler_config=vllm_config.scheduler_config,
lora_config=vllm_config.lora_config) # type: ignore[arg-type] lora_config=vllm_config.lora_config) # type: ignore[arg-type]
self.disable_any_whitespace = False self.disable_any_whitespace = \
backend_options = GuidedDecodingParams( vllm_config.decoding_config.disable_any_whitespace
backend=vllm_config.decoding_config.guided_decoding_backend
).backend_options()
for option in backend_options:
if option == "disable-any-whitespace":
self.disable_any_whitespace = True
else:
raise ValueError(
f"Unsupported option for the xgrammar backend: {option}")
tokenizer = tokenizer_group.get_lora_tokenizer(None) tokenizer = tokenizer_group.get_lora_tokenizer(None)
self.vocab_size = vllm_config.model_config.get_vocab_size() self.vocab_size = vllm_config.model_config.get_vocab_size()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment