Unverified Commit 1f106ee3 authored by Xinyuan Tong's avatar Xinyuan Tong Committed by GitHub
Browse files

[grammar] Avoid server crash when grammar backend is None (#11401)

parent 9b8ebb27
...@@ -1442,26 +1442,29 @@ class Scheduler( ...@@ -1442,26 +1442,29 @@ class Scheduler(
or req.sampling_params.ebnf is not None or req.sampling_params.ebnf is not None
or req.sampling_params.structural_tag is not None or req.sampling_params.structural_tag is not None
): ):
assert self.grammar_backend is not None if self.grammar_backend is None:
if req.sampling_params.json_schema is not None: error_msg = "Grammar-based generation (json_schema, regex, ebnf, structural_tag) is not supported when the server is launched with --grammar-backend none"
key = ("json", req.sampling_params.json_schema) req.set_finish_with_abort(error_msg)
elif req.sampling_params.regex is not None:
key = ("regex", req.sampling_params.regex)
elif req.sampling_params.ebnf is not None:
key = ("ebnf", req.sampling_params.ebnf)
elif req.sampling_params.structural_tag:
key = ("structural_tag", req.sampling_params.structural_tag)
value, cache_hit = self.grammar_backend.get_cached_or_future_value(key)
req.grammar = value
if not cache_hit:
req.grammar_key = key
add_to_grammar_queue = True
else: else:
if value is INVALID_GRAMMAR_OBJ: # We hit a cached invalid grammar. if req.sampling_params.json_schema is not None:
error_msg = f"Invalid grammar request with cache hit: {key=}" key = ("json", req.sampling_params.json_schema)
req.set_finish_with_abort(error_msg) elif req.sampling_params.regex is not None:
key = ("regex", req.sampling_params.regex)
elif req.sampling_params.ebnf is not None:
key = ("ebnf", req.sampling_params.ebnf)
elif req.sampling_params.structural_tag:
key = ("structural_tag", req.sampling_params.structural_tag)
value, cache_hit = self.grammar_backend.get_cached_or_future_value(key)
req.grammar = value
if not cache_hit:
req.grammar_key = key
add_to_grammar_queue = True
else:
if value is INVALID_GRAMMAR_OBJ: # We hit a cached invalid grammar.
error_msg = f"Invalid grammar request with cache hit: {key=}"
req.set_finish_with_abort(error_msg)
if add_to_grammar_queue: if add_to_grammar_queue:
self.grammar_queue.append(req) self.grammar_queue.append(req)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment