Unverified Commit a36c6758 authored by Maximilien de Bayser's avatar Maximilien de Bayser Committed by GitHub
Browse files

Don't skip special tokens with hermes-style tool calling (#25281)


Signed-off-by: default avatarMax de Bayser <mbayser@br.ibm.com>
parent 3da17c2c
......@@ -98,6 +98,15 @@ class Hermes2ProToolParser(ToolParser):
else:
return delta_text
def adjust_request(
self, request: ChatCompletionRequest) -> ChatCompletionRequest:
if request.tools and request.tool_choice != 'none':
# do not skip special tokens because the tool_call tokens are
# marked "special" in some models. Since they are skipped
# prior to the call to the tool parser, it breaks tool calling.
request.skip_special_tokens = False
return request
def extract_tool_calls(
self,
model_output: str,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment