Skip to content

GitLab

"megatron/vscode:/vscode.git/clone" did not exist on "018391a658f3a946d24dfb0d887f52030e998271"
Unverified Commit c1c8dd1d authored by Chang Su's avatar Chang Su Committed by GitHub
Browse files

[router][tool parser] Modify tool parser to return both normal text and tool... 

[router][tool parser] Modify tool parser to return both normal text and tool calls (non-stream) (#10995)
parent f6bc3f52
Hide whitespace changes
Inline Side-by-side
Showing with 391 additions and 391 deletions
sgl-router/tests/tool_parser_json.rs
View file @ c1c8dd1d
......@@ -10,11 +10,11 @@ async fn test_simple_json_tool_call() {
let parser = JsonParser::new();
let input = r#"{"name": "get_weather", "arguments": {"location": "San Francisco"}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["location"], "San Francisco");
}
......@@ -26,10 +26,10 @@ async fn test_json_array_of_tools() {
{"name": "search", "arguments": {"query": "news"}}
]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "get_weather");
assert_eq!(result[1].function.name, "search");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "get_weather");
assert_eq!(tools[1].function.name, "search");
}
#[tokio::test]
......@@ -37,11 +37,11 @@ async fn test_json_with_parameters_key() {
let parser = JsonParser::new();
let input = r#"{"name": "calculate", "parameters": {"x": 10, "y": 20}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "calculate");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "calculate");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["x"], 10);
assert_eq!(args["y"], 20);
}
......@@ -51,9 +51,9 @@ async fn test_json_extraction_from_text() {
let parser = JsonParser::new();
let input = r#"I'll help you with that. {"name": "search", "arguments": {"query": "rust"}} Let me search for that."#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "search");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "search");
}
#[tokio::test]
......@@ -73,11 +73,11 @@ async fn test_json_with_nested_objects() {
}
}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "update_config");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "update_config");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["settings"]["theme"], "dark");
assert_eq!(args["settings"]["notifications"]["email"], true);
}
......@@ -87,10 +87,10 @@ async fn test_json_with_special_characters() {
let parser = JsonParser::new();
let input = r#"{"name": "echo", "arguments": {"text": "Line 1\nLine 2\tTabbed", "path": "C:\\Users\\test"}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "Line 1\nLine 2\tTabbed");
assert_eq!(args["path"], "C:\\Users\\test");
}
......@@ -100,10 +100,10 @@ async fn test_json_with_unicode() {
let parser = JsonParser::new();
let input = r#"{"name": "translate", "arguments": {"text": "Hello 世界 🌍", "emoji": "😊"}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "Hello 世界 🌍");
assert_eq!(args["emoji"], "😊");
}
......@@ -113,11 +113,11 @@ async fn test_json_empty_arguments() {
let parser = JsonParser::new();
let input = r#"{"name": "ping", "arguments": {}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "ping");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "ping");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args, json!({}));
}
......@@ -127,13 +127,13 @@ async fn test_json_invalid_format() {
// Missing closing brace
let input = r#"{"name": "test", "arguments": {"key": "value""#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 0);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
// Not JSON at all
let input = "This is just plain text";
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 0);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
}
#[tokio::test]
......
sgl-router/tests/tool_parser_kimik2.rs
View file @ c1c8dd1d
......@@ -12,11 +12,11 @@ async fn test_kimik2_complete_parsing() {
<|tool_calls_section_end|>
The weather in Tokyo is..."#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["location"], "Tokyo");
assert_eq!(args["units"], "celsius");
}
......@@ -30,10 +30,10 @@ async fn test_kimik2_multiple_tools() {
<|tool_call_begin|>functions.translate:1<|tool_call_argument_begin|>{"text": "Hello", "to": "ja"}<|tool_call_end|>
<|tool_calls_section_end|>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "search");
assert_eq!(result[1].function.name, "translate");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "translate");
}
#[tokio::test]
......@@ -44,11 +44,11 @@ async fn test_kimik2_with_whitespace() {
<|tool_call_begin|> functions.test:0 <|tool_call_argument_begin|> {"key": "value", "num": 42} <|tool_call_end|>
<|tool_calls_section_end|>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["key"], "value");
assert_eq!(args["num"], 42);
}
......@@ -117,11 +117,11 @@ async fn test_kimik2_sequential_indices() {
<|tool_call_begin|>functions.third:2<|tool_call_argument_begin|>{"param": "c"}<|tool_call_end|>
<|tool_calls_section_end|>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 3);
assert_eq!(result[0].function.name, "first");
assert_eq!(result[1].function.name, "second");
assert_eq!(result[2].function.name, "third");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 3);
assert_eq!(tools[0].function.name, "first");
assert_eq!(tools[1].function.name, "second");
assert_eq!(tools[2].function.name, "third");
}
#[tokio::test]
......@@ -134,10 +134,10 @@ async fn test_function_index_extraction() {
<|tool_call_begin|>functions.calc:1<|tool_call_argument_begin|>{"x": 10}<|tool_call_end|>
<|tool_calls_section_end|>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "search");
assert_eq!(result[1].function.name, "calc");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "calc");
// TODO: Verify indices are preserved: 0 and 1
// TODO: Verify normal text = "Text before tool calls."
}
......@@ -150,7 +150,7 @@ async fn test_namespace_extraction() {
<|tool_call_begin|>api.tools.search:0<|tool_call_argument_begin|>{"q": "test"}<|tool_call_end|>
<|tool_calls_section_end|>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "search"); // Should extract after last dot
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "search"); // Should extract after last dot
}
sgl-router/tests/tool_parser_llama.rs
View file @ c1c8dd1d
......@@ -9,11 +9,11 @@ async fn test_llama_python_tag_format() {
let parser = LlamaParser::new();
let input = r#"<|python_tag|>{"name": "search", "arguments": {"query": "weather"}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "search");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "search");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["query"], "weather");
}
......@@ -22,11 +22,11 @@ async fn test_llama_plain_json_fallback() {
let parser = LlamaParser::new();
let input = r#"{"name": "calculate", "arguments": {"x": 5, "y": 10}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "calculate");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "calculate");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["x"], 5);
assert_eq!(args["y"], 10);
}
......@@ -36,11 +36,11 @@ async fn test_llama_with_text_before() {
let parser = LlamaParser::new();
let input = r#"Let me help you with that. <|python_tag|>{"name": "get_time", "arguments": {"timezone": "UTC"}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_time");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_time");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["timezone"], "UTC");
}
......@@ -58,11 +58,11 @@ async fn test_llama_with_nested_json() {
}
}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "update_settings");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "update_settings");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["preferences"]["theme"], "dark");
assert_eq!(args["notifications"], true);
}
......@@ -73,15 +73,15 @@ async fn test_llama_empty_arguments() {
// With python_tag
let input = r#"<|python_tag|>{"name": "ping", "arguments": {}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "ping");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "ping");
// Plain JSON
let input = r#"{"name": "ping", "arguments": {}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "ping");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "ping");
}
#[tokio::test]
......@@ -99,8 +99,8 @@ async fn test_llama_invalid_json_after_tag() {
let parser = LlamaParser::new();
let input = r#"<|python_tag|>{"name": invalid}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 0);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
}
#[tokio::test]
......@@ -112,9 +112,9 @@ async fn test_llama_real_world_output() {
<|python_tag|>{"name": "web_search", "arguments": {"query": "Llama 3.2 model capabilities", "num_results": 5, "search_type": "recent"}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "web_search");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "web_search");
let formatted_input = r#"<|python_tag|>{
"name": "get_current_time",
......@@ -124,9 +124,9 @@ async fn test_llama_real_world_output() {
}
}"#;
let result2 = parser.parse_complete(formatted_input).await.unwrap();
assert_eq!(result2.len(), 1);
assert_eq!(result2[0].function.name, "get_current_time");
let (_normal_text, tools2) = parser.parse_complete(formatted_input).await.unwrap();
assert_eq!(tools2.len(), 1);
assert_eq!(tools2[0].function.name, "get_current_time");
}
#[tokio::test]
......@@ -136,9 +136,9 @@ async fn test_llama_json_array_format() {
// Plain JSON array (should work as fallback)
let input = r#"[{"name": "func1", "arguments": {}}, {"name": "func2", "arguments": {}}]"#;
let result = parser.parse_complete(input).await.unwrap();
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
// Current implementation might handle this through JSON fallback
assert!(!result.is_empty());
assert!(!tools.is_empty());
}
#[tokio::test]
......@@ -146,11 +146,11 @@ async fn test_single_json() {
let parser = LlamaParser::new();
let text = r#"{"name": "get_weather", "arguments": {"city": "Paris"}}"#;
let result = parser.parse_complete(text).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["city"], "Paris");
}
......@@ -159,10 +159,10 @@ async fn test_multiple_json_with_separator() {
let parser = LlamaParser::new();
let text = r#"<|python_tag|>{"name": "get_weather", "arguments": {"city": "Paris"}};{"name": "get_tourist_attractions", "arguments": {"city": "Paris"}}"#;
let result = parser.parse_complete(text).await.unwrap();
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
// Note: Current implementation may only parse the first one due to semicolon handling
assert!(!result.is_empty());
assert_eq!(result[0].function.name, "get_weather");
assert!(!tools.is_empty());
assert_eq!(tools[0].function.name, "get_weather");
}
#[tokio::test]
......@@ -170,10 +170,10 @@ async fn test_multiple_json_with_separator_customized() {
let parser = LlamaParser::new();
let text = r#"<|python_tag|>{"name": "get_weather", "arguments": {}}<|python_tag|>{"name": "get_tourist_attractions", "arguments": {}}"#;
let result = parser.parse_complete(text).await.unwrap();
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
// Current implementation may handle this differently
assert!(!result.is_empty());
assert_eq!(result[0].function.name, "get_weather");
assert!(!tools.is_empty());
assert_eq!(tools[0].function.name, "get_weather");
}
#[tokio::test]
......@@ -181,9 +181,9 @@ async fn test_json_with_trailing_text() {
let parser = LlamaParser::new();
let text = r#"{"name": "get_weather", "arguments": {}} Some follow-up text"#;
let result = parser.parse_complete(text).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
}
#[tokio::test]
......@@ -191,10 +191,10 @@ async fn test_invalid_then_valid_json() {
let parser = LlamaParser::new();
let text = r#"{"name": "get_weather", "arguments": {{"name": "get_weather", "arguments": {}}"#;
let result = parser.parse_complete(text).await.unwrap();
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
// Should parse at least one valid JSON
if !result.is_empty() {
assert_eq!(result[0].function.name, "get_weather");
if !tools.is_empty() {
assert_eq!(tools[0].function.name, "get_weather");
}
}
......@@ -203,8 +203,8 @@ async fn test_plain_text_only() {
let parser = LlamaParser::new();
let text = "This is just plain explanation text.";
let result = parser.parse_complete(text).await.unwrap();
assert_eq!(result.len(), 0);
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
assert_eq!(tools.len(), 0);
}
#[tokio::test]
......@@ -212,9 +212,9 @@ async fn test_with_python_tag_prefix() {
let parser = LlamaParser::new();
let text = r#"Some intro. <|python_tag|>{"name": "get_weather", "arguments": {}}"#;
let result = parser.parse_complete(text).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
}
// STREAMING TESTS
......
sgl-router/tests/tool_parser_mistral.rs
View file @ c1c8dd1d
......@@ -11,11 +11,11 @@ async fn test_mistral_single_tool() {
let input = r#"Let me search for that.
[TOOL_CALLS] [{"name": "search_web", "arguments": {"query": "latest news", "max_results": 5}}]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "search_web");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "search_web");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["query"], "latest news");
assert_eq!(args["max_results"], 5);
}
......@@ -29,15 +29,15 @@ async fn test_mistral_multiple_tools() {
{"name": "search_news", "arguments": {"query": "AI developments", "limit": 10}}
]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(result[0].function.name, "get_weather");
let args0: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
assert_eq!(tools[0].function.name, "get_weather");
let args0: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args0["city"], "Tokyo");
assert_eq!(result[1].function.name, "search_news");
let args1: serde_json::Value = serde_json::from_str(&result[1].function.arguments).unwrap();
assert_eq!(tools[1].function.name, "search_news");
let args1: serde_json::Value = serde_json::from_str(&tools[1].function.arguments).unwrap();
assert_eq!(args1["query"], "AI developments");
}
......@@ -47,10 +47,10 @@ async fn test_mistral_nested_json() {
let input = r#"Processing complex data.
[TOOL_CALLS] [{"name": "process_data", "arguments": {"config": {"nested": {"value": [1, 2, 3]}}, "enabled": true}}]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["config"]["nested"]["value"], json!([1, 2, 3]));
assert_eq!(args["enabled"], true);
}
......@@ -62,9 +62,9 @@ async fn test_mistral_with_text_after() {
And here's some text after the tool call that should be ignored."#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
}
#[tokio::test]
......@@ -72,9 +72,9 @@ async fn test_mistral_empty_arguments() {
let parser = MistralParser::new();
let input = r#"[TOOL_CALLS] [{"name": "ping", "arguments": {}}]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "ping");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "ping");
}
#[tokio::test]
......@@ -82,10 +82,10 @@ async fn test_mistral_with_brackets_in_strings() {
let parser = MistralParser::new();
let input = r#"[TOOL_CALLS] [{"name": "echo", "arguments": {"text": "Array notation: arr[0] = value[1]"}}]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "Array notation: arr[0] = value[1]");
}
......@@ -105,15 +105,15 @@ async fn test_mistral_malformed_json() {
// Missing closing bracket
let input = r#"[TOOL_CALLS] [{"name": "test", "arguments": {}"#;
if let Ok(result) = parser.parse_complete(input).await {
assert_eq!(result.len(), 0);
if let Ok((_normal_text, tools)) = parser.parse_complete(input).await {
assert_eq!(tools.len(), 0);
}
// Error is also acceptable for malformed input
// Invalid JSON inside
let input = r#"[TOOL_CALLS] [{"name": invalid}]"#;
if let Ok(result) = parser.parse_complete(input).await {
assert_eq!(result.len(), 0);
if let Ok((_normal_text, tools)) = parser.parse_complete(input).await {
assert_eq!(tools.len(), 0);
}
// Error is also acceptable for malformed input
}
......@@ -146,8 +146,8 @@ async fn test_mistral_real_world_output() {
Let me execute these searches for you."#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "web_search");
assert_eq!(result[1].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "web_search");
assert_eq!(tools[1].function.name, "get_weather");
}
sgl-router/tests/tool_parser_mixed_edge_cases.rs
View file @ c1c8dd1d
......@@ -17,9 +17,9 @@ async fn test_mixed_formats_in_text() {
But here's the actual JSON: {"name": "test", "arguments": {}}
"#;
let result = json_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
// Mistral parser should ignore JSON and other formats
let mistral_parser = MistralParser::new();
......@@ -28,9 +28,9 @@ async fn test_mixed_formats_in_text() {
[TOOL_CALLS] [{"name": "real", "arguments": {}}]
"#;
let result = mistral_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "real");
let (_normal_text, tools) = mistral_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "real");
}
#[tokio::test]
......@@ -38,9 +38,9 @@ async fn test_format_markers_in_string_content() {
let pythonic_parser = PythonicParser::new();
let input = r#"[echo(text="Use [TOOL_CALLS] and <tool_call> in text")]"#;
let result = pythonic_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let (_normal_text, tools) = pythonic_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "Use [TOOL_CALLS] and <tool_call> in text");
let qwen_parser = QwenParser::new();
......@@ -48,9 +48,9 @@ async fn test_format_markers_in_string_content() {
{"name": "log", "arguments": {"msg": "Found [function()] pattern"}}
</tool_call>"#;
let result = qwen_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let (_normal_text, tools) = qwen_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["msg"], "Found [function()] pattern");
}
......@@ -75,11 +75,11 @@ async fn test_deeply_nested_json_structures() {
}
}"#;
let result = json_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "deep_process");
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "deep_process");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert!(args["level1"]["level2"]["level3"]["level4"]["level5"]["data"].is_array());
}
......@@ -93,14 +93,14 @@ async fn test_multiple_sequential_calls_different_formats() {
// Llama parser currently only returns the first tool found
let input = r#"First call: <|python_tag|>{"name": "call1", "arguments": {}}"#;
let result = llama_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "call1");
let (_normal_text, tools) = llama_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "call1");
let input2 = r#"{"name": "call2", "arguments": {"x": 1}}"#;
let result2 = llama_parser.parse_complete(input2).await.unwrap();
assert_eq!(result2.len(), 1);
assert_eq!(result2[0].function.name, "call2");
let (_normal_text2, tools2) = llama_parser.parse_complete(input2).await.unwrap();
assert_eq!(tools2.len(), 1);
assert_eq!(tools2[0].function.name, "call2");
}
#[tokio::test]
......@@ -119,8 +119,8 @@ async fn test_empty_and_whitespace_variations() {
];
for input in cases {
let result = json_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1, "Should parse regardless of whitespace");
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1, "Should parse regardless of whitespace");
}
}
......@@ -141,11 +141,11 @@ async fn test_special_json_values() {
}
}"#;
let result = json_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test_special");
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test_special");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert!(args["special_strings"].is_array());
assert!(args["escaped"].is_string());
}
......@@ -181,22 +181,22 @@ async fn test_boundary_cases_for_extraction() {
// JSON at the very beginning
let input = r#"{"name": "start", "arguments": {}} and then text"#;
let result = json_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "start");
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "start");
// JSON at the very end
let input = r#"Some text first {"name": "end", "arguments": {}}"#;
let result = json_parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "end");
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "end");
// Multiple JSON objects in text (should find first valid one)
let input =
r#"Text {"name": "first", "arguments": {}} more {"name": "second", "arguments": {}}"#;
let result = json_parser.parse_complete(input).await.unwrap();
assert!(!result.is_empty());
assert_eq!(result[0].function.name, "first");
let (_normal_text, tools) = json_parser.parse_complete(input).await.unwrap();
assert!(!tools.is_empty());
assert_eq!(tools[0].function.name, "first");
}
#[tokio::test]
......@@ -205,15 +205,15 @@ async fn test_pythonic_edge_cases() {
// Function name with underscores and numbers
let input = r#"[func_name_2(param_1="value")]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "func_name_2");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "func_name_2");
// Empty string argument
let input = r#"[process(text="")]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "");
}
......@@ -238,11 +238,11 @@ async fn test_mistral_with_pretty_json() {
}
]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "formatted");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "formatted");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["nested"]["key"], "value");
assert_eq!(args["array"], json!([1, 2, 3]));
}
......@@ -256,11 +256,11 @@ async fn test_qwen_with_cdata_like_content() {
{"name": "process", "arguments": {"xml": "<![CDATA[some data]]>"}}
</tool_call>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "process");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "process");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["xml"], "<![CDATA[some data]]>");
}
......@@ -271,9 +271,9 @@ async fn test_extremely_long_function_names() {
let long_name = "very_long_function_name_that_might_appear_in_generated_code_somewhere";
let input = format!(r#"[{}(param="value")]"#, long_name);
let result = parser.parse_complete(&input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, long_name);
let (_normal_text, tools) = parser.parse_complete(&input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, long_name);
}
#[tokio::test]
......@@ -283,10 +283,10 @@ async fn test_json_with_duplicate_keys() {
// JSON with duplicate keys (last one should win per JSON spec)
let input = r#"{"name": "test", "arguments": {"key": "first", "key": "second"}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
// JSON parsers typically keep the last value for duplicate keys
assert_eq!(args["key"], "second");
}
sgl-router/tests/tool_parser_pythonic.rs
View file @ c1c8dd1d
......@@ -10,11 +10,11 @@ async fn test_pythonic_single_function() {
let parser = PythonicParser::new();
let input = r#"[get_weather(city="London", units="celsius")]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["city"], "London");
assert_eq!(args["units"], "celsius");
}
......@@ -25,12 +25,12 @@ async fn test_pythonic_multiple_functions() {
let input =
r#"[search_web(query="Rust programming", max_results=5), get_time(timezone="UTC")]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "search_web");
assert_eq!(result[1].function.name, "get_time");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "search_web");
assert_eq!(tools[1].function.name, "get_time");
let args0: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args0: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args0["query"], "Rust programming");
assert_eq!(args0["max_results"], 5);
}
......@@ -40,10 +40,10 @@ async fn test_pythonic_with_python_literals() {
let parser = PythonicParser::new();
let input = r#"[configure(enabled=True, disabled=False, optional=None)]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["enabled"], true);
assert_eq!(args["disabled"], false);
assert_eq!(args["optional"], json!(null));
......@@ -55,10 +55,10 @@ async fn test_pythonic_with_lists_and_dicts() {
let input =
r#"[process_data(items=[1, 2, 3], config={"key": "value", "nested": {"deep": True}})]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["items"], json!([1, 2, 3]));
assert_eq!(args["config"]["key"], "value");
assert_eq!(args["config"]["nested"]["deep"], true);
......@@ -71,11 +71,11 @@ async fn test_pythonic_with_special_tokens() {
// Llama 4 sometimes outputs these tokens
let input = r#"<|python_start|>[calculate(x=10, y=20)]<|python_end|>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "calculate");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "calculate");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["x"], 10);
assert_eq!(args["y"], 20);
}
......@@ -85,10 +85,10 @@ async fn test_pythonic_with_nested_parentheses() {
let parser = PythonicParser::new();
let input = r#"[math_eval(expression="(2 + 3) * (4 - 1)", round_to=2)]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["expression"], "(2 + 3) * (4 - 1)");
assert_eq!(args["round_to"], 2);
}
......@@ -98,10 +98,10 @@ async fn test_pythonic_with_escaped_quotes() {
let parser = PythonicParser::new();
let input = r#"[echo(text="She said \"Hello\" to him")]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "She said \"Hello\" to him");
}
......@@ -110,11 +110,11 @@ async fn test_pythonic_empty_arguments() {
let parser = PythonicParser::new();
let input = r#"[ping()]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "ping");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "ping");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args, json!({}));
}
......@@ -135,8 +135,8 @@ async fn test_pythonic_invalid_syntax() {
// Missing closing bracket
let input = r#"[function(arg=value"#;
if let Ok(result) = parser.parse_complete(input).await {
assert_eq!(result.len(), 0);
if let Ok((_normal_text, tools)) = parser.parse_complete(input).await {
assert_eq!(tools.len(), 0);
}
// Error is also acceptable for invalid syntax
......@@ -144,10 +144,10 @@ async fn test_pythonic_invalid_syntax() {
// Note: The parser currently accepts this invalid syntax and returns a result
// This is a known limitation of the current implementation
let input = r#"[function(=value)]"#;
if let Ok(result) = parser.parse_complete(input).await {
if let Ok((_normal_text, tools)) = parser.parse_complete(input).await {
// The parser incorrectly accepts this, returning 1 result
// We'll accept this behavior for now but note it's not ideal
assert!(result.len() <= 1, "Should parse at most one function");
assert!(tools.len() <= 1, "Should parse at most one function");
}
// Error would be the correct behavior
}
......@@ -165,13 +165,13 @@ async fn test_pythonic_real_world_llama4() {
These functions will provide the information you need."#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 3);
assert_eq!(result[0].function.name, "web_search");
assert_eq!(result[1].function.name, "calculate");
assert_eq!(result[2].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 3);
assert_eq!(tools[0].function.name, "web_search");
assert_eq!(tools[1].function.name, "calculate");
assert_eq!(tools[2].function.name, "get_weather");
let args0: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args0: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args0["query"], "latest Rust features");
assert_eq!(args0["safe_search"], true);
}
......@@ -182,11 +182,11 @@ async fn test_pythonic_nested_brackets_in_lists() {
let input = r#"[process_matrix(data=[[1, 2], [3, 4]], labels=["row[0]", "row[1]"])]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "process_matrix");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "process_matrix");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["data"], json!([[1, 2], [3, 4]]));
assert_eq!(args["labels"], json!(["row[0]", "row[1]"]));
}
......@@ -198,11 +198,11 @@ async fn test_pythonic_nested_brackets_in_dicts() {
let input =
r#"[analyze(config={"patterns": ["[a-z]+", "[0-9]+"], "nested": {"list": [1, [2, 3]]}})]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "analyze");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "analyze");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["config"]["patterns"], json!(["[a-z]+", "[0-9]+"]));
assert_eq!(args["config"]["nested"]["list"], json!([1, [2, 3]]));
}
......@@ -213,11 +213,11 @@ async fn test_pythonic_mixed_quotes() {
let input = r#"[format_text(single='Hello', double="World", mixed="It's \"quoted\"")]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "format_text");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "format_text");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["single"], "Hello");
assert_eq!(args["double"], "World");
assert_eq!(args["mixed"], "It's \"quoted\"");
......@@ -233,11 +233,11 @@ async fn test_pythonic_complex_nesting() {
metadata={"tags": ["nested[0]", "nested[1]"], "config": {"depth": [1, 2, 3]}}
)]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "transform");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "transform");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert!(args["matrix"].is_array());
assert!(args["operations"].is_array());
assert_eq!(args["operations"][0]["type"], "scale");
......@@ -530,12 +530,12 @@ async fn test_detect_and_parse_with_python_start_and_end_token() {
let parser = PythonicParser::new();
let text = "User wants to get the weather in Mars. <|python_start|>[get_weather(location='Mars', unit='celsius')]<|python_end|> In this way we will get the weather in Mars.";
let result = parser.parse_complete(text).await.unwrap();
let (_normal_text, tools) = parser.parse_complete(text).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_weather");
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["location"], "Mars");
assert_eq!(args["unit"], "celsius");
}
sgl-router/tests/tool_parser_qwen.rs
View file @ c1c8dd1d
......@@ -12,11 +12,11 @@ async fn test_qwen_single_tool() {
{"name": "get_weather", "arguments": {"city": "Beijing", "units": "celsius"}}
</tool_call>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["city"], "Beijing");
assert_eq!(args["units"], "celsius");
}
......@@ -32,10 +32,10 @@ async fn test_qwen_multiple_sequential_tools() {
{"name": "translate", "arguments": {"text": "Hello", "to": "zh"}}
</tool_call>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "search");
assert_eq!(result[1].function.name, "translate");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "translate");
}
#[tokio::test]
......@@ -55,11 +55,11 @@ async fn test_qwen_pretty_printed_json() {
}
</tool_call>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "create_document");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "create_document");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["metadata"]["author"], "Qwen");
assert_eq!(args["metadata"]["tags"], json!(["test", "example"]));
}
......@@ -79,10 +79,10 @@ Now I'll translate something.
</tool_call>
Done!"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "search");
assert_eq!(result[1].function.name, "translate");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "search");
assert_eq!(tools[1].function.name, "translate");
}
#[tokio::test]
......@@ -92,9 +92,9 @@ async fn test_qwen_empty_arguments() {
{"name": "get_time", "arguments": {}}
</tool_call>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_time");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_time");
}
#[tokio::test]
......@@ -104,10 +104,10 @@ async fn test_qwen_with_newlines_in_strings() {
{"name": "write_file", "arguments": {"content": "Line 1\nLine 2\nLine 3", "path": "/tmp/test.txt"}}
</tool_call>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["content"], "Line 1\nLine 2\nLine 3");
}
......@@ -128,14 +128,14 @@ async fn test_qwen_incomplete_tags() {
// Missing closing tag
let input = r#"<tool_call>
{"name": "test", "arguments": {}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 0);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
// Missing opening tag
let input = r#"{"name": "test", "arguments": {}}
</tool_call>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 0);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0);
}
#[tokio::test]
......@@ -171,12 +171,12 @@ Let me also calculate something for you:
These tools will provide the information you need."#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "web_search");
assert_eq!(result[1].function.name, "calculator");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "web_search");
assert_eq!(tools[1].function.name, "calculator");
let args0: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args0: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args0["query"], "quantum computing breakthroughs 2024");
assert_eq!(args0["safe_search"], true);
}
......
sgl-router/tests/tool_parser_registry.rs
View file @ c1c8dd1d
......@@ -24,9 +24,9 @@ async fn test_openai_models_use_json() {
for model in models {
let parser = registry.get_parser(model).unwrap();
let test_input = r#"{"name": "test", "arguments": {}}"#;
let result = parser.parse_complete(test_input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(test_input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
}
}
......@@ -38,8 +38,8 @@ async fn test_anthropic_models_use_json() {
for model in models {
let parser = registry.get_parser(model).unwrap();
let test_input = r#"{"name": "test", "arguments": {}}"#;
let result = parser.parse_complete(test_input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(test_input).await.unwrap();
assert_eq!(tools.len(), 1);
}
}
......@@ -51,9 +51,9 @@ async fn test_mistral_models() {
for model in models {
let parser = registry.get_parser(model).unwrap();
let test_input = r#"[TOOL_CALLS] [{"name": "test", "arguments": {}}]"#;
let result = parser.parse_complete(test_input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(test_input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
}
}
......@@ -67,9 +67,9 @@ async fn test_qwen_models() {
let test_input = r#"<tool_call>
{"name": "test", "arguments": {}}
</tool_call>"#;
let result = parser.parse_complete(test_input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(test_input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
}
}
......@@ -80,22 +80,22 @@ async fn test_llama_model_variants() {
// Llama 4 uses pythonic
let parser = registry.get_parser("llama-4-70b").unwrap();
let test_input = r#"[get_weather(city="NYC")]"#;
let result = parser.parse_complete(test_input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "get_weather");
let (_normal_text, tools) = parser.parse_complete(test_input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "get_weather");
// Llama 3.2 uses python_tag
let parser = registry.get_parser("llama-3.2-8b").unwrap();
let test_input = r#"<|python_tag|>{"name": "test", "arguments": {}}"#;
let result = parser.parse_complete(test_input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(test_input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
// Other Llama models use JSON
let parser = registry.get_parser("llama-2-70b").unwrap();
let test_input = r#"{"name": "test", "arguments": {}}"#;
let result = parser.parse_complete(test_input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(test_input).await.unwrap();
assert_eq!(tools.len(), 1);
}
#[tokio::test]
......@@ -105,9 +105,9 @@ async fn test_deepseek_models() {
// DeepSeek uses pythonic format (simplified, v3 would need custom parser)
let parser = registry.get_parser("deepseek-coder").unwrap();
let test_input = r#"[function(arg="value")]"#;
let result = parser.parse_complete(test_input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "function");
let (_normal_text, tools) = parser.parse_complete(test_input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "function");
}
#[tokio::test]
......@@ -117,9 +117,9 @@ async fn test_unknown_model_fallback() {
// Unknown models should fall back to JSON parser
let parser = registry.get_parser("unknown-model-xyz").unwrap();
let test_input = r#"{"name": "fallback", "arguments": {}}"#;
let result = parser.parse_complete(test_input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "fallback");
let (_normal_text, tools) = parser.parse_complete(test_input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "fallback");
}
#[tokio::test]
......@@ -181,10 +181,10 @@ The weather information has been requested."#,
for (model, output, expected_name) in test_cases {
let parser = registry.get_parser(model).unwrap();
let result = parser.parse_complete(output).await.unwrap();
assert!(!result.is_empty(), "No tools parsed for model {}", model);
let (_normal_text, tools) = parser.parse_complete(output).await.unwrap();
assert!(!tools.is_empty(), "No tools parsed for model {}", model);
assert_eq!(
result[0].function.name, expected_name,
tools[0].function.name, expected_name,
"Wrong function name for model {}",
model
);
......
sgl-router/tests/tool_parser_step3.rs
View file @ c1c8dd1d
......@@ -15,11 +15,11 @@ async fn test_step3_complete_parsing() {
<｜tool_calls_end｜>
Here are the results..."#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "search");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "search");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["query"], "rust programming");
assert_eq!(args["limit"], 10);
}
......@@ -38,10 +38,10 @@ async fn test_step3_multiple_tools() {
</steptml:invoke><｜tool_call_end｜>
<｜tool_calls_end｜>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "get_weather");
assert_eq!(result[1].function.name, "get_news");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "get_weather");
assert_eq!(tools[1].function.name, "get_news");
}
#[tokio::test]
......@@ -58,10 +58,10 @@ async fn test_step3_type_conversion() {
</steptml:invoke><｜tool_call_end｜>
<｜tool_calls_end｜>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["count"], 100);
assert_eq!(args["rate"], 2.5);
assert_eq!(args["active"], true);
......@@ -132,11 +132,11 @@ async fn test_step3_nested_steptml() {
</steptml:invoke><｜tool_call_end｜>
<｜tool_calls_end｜>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "config");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "config");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert!(args["settings"].is_object());
assert!(args["array"].is_array());
}
......@@ -153,10 +153,10 @@ async fn test_step3_python_literals() {
</steptml:invoke><｜tool_call_end｜>
<｜tool_calls_end｜>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["bool_true"], true);
assert_eq!(args["bool_false"], false);
assert_eq!(args["none_value"], serde_json::Value::Null);
......@@ -174,11 +174,11 @@ async fn test_steptml_format() {
</steptml:invoke><｜tool_call_end｜>
<｜tool_calls_end｜>Text after."#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "search");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "search");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["query"], "rust lang");
assert_eq!(args["limit"], 10);
// TODO: Verify normal text extraction
......@@ -195,10 +195,10 @@ async fn test_json_parameter_values() {
</steptml:invoke><｜tool_call_end｜>
<｜tool_calls_end｜>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert!(args["settings"].is_object());
assert!(args["items"].is_array());
}
......@@ -214,11 +214,11 @@ async fn test_step3_parameter_with_angle_brackets() {
</steptml:invoke><｜tool_call_end｜>
<｜tool_calls_end｜>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "compare");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "compare");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["expression"], "a < b && b > c");
assert_eq!(args["context"], "comparison test");
}
......@@ -233,6 +233,6 @@ async fn test_step3_empty_function_name() {
</steptml:invoke><｜tool_call_end｜>
<｜tool_calls_end｜>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 0); // Should reject empty function name
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0); // Should reject empty function name
}
sgl-router/tests/tool_parser_wrapper_tokens.rs
View file @ c1c8dd1d
......@@ -15,11 +15,11 @@ async fn test_json_with_xml_style_wrapper() {
let input =
r#"Some text before <tool>{"name": "test", "arguments": {"x": 1}}</tool> and after"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["x"], 1);
}
......@@ -32,14 +32,14 @@ async fn test_json_with_multiple_wrapper_pairs() {
});
let input1 = r#"<tool>{"name": "tool1", "arguments": {}}</tool>"#;
let result1 = parser.parse_complete(input1).await.unwrap();
assert_eq!(result1.len(), 1);
assert_eq!(result1[0].function.name, "tool1");
let (_normal_text, tools1) = parser.parse_complete(input1).await.unwrap();
assert_eq!(tools1.len(), 1);
assert_eq!(tools1[0].function.name, "tool1");
let input2 = r#"<<TOOL>>{"name": "tool2", "arguments": {}}<</TOOL>>"#;
let result2 = parser.parse_complete(input2).await.unwrap();
assert_eq!(result2.len(), 1);
assert_eq!(result2[0].function.name, "tool2");
let (_normal_text, tools2) = parser.parse_complete(input2).await.unwrap();
assert_eq!(tools2.len(), 1);
assert_eq!(tools2[0].function.name, "tool2");
}
#[tokio::test]
......@@ -52,9 +52,9 @@ async fn test_json_with_only_start_token() {
let input = r#"Some preamble >>>FUNCTION:{"name": "execute", "arguments": {"cmd": "ls"}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "execute");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "execute");
}
#[tokio::test]
......@@ -68,9 +68,9 @@ async fn test_json_with_custom_separator() {
// Though we're not testing multiple tools here, the separator is configured
let input = r#"[FUNC]{"name": "test", "arguments": {}}[/FUNC]"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
}
#[tokio::test]
......@@ -88,21 +88,21 @@ async fn test_json_with_nested_wrapper_tokens_in_content() {
let input =
r#"<call>{"name": "echo", "arguments": {"text": "Use <call> and </call> tags"}}</call>"#;
let result = parser.parse_complete(input).await.unwrap();
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
// This is a known limitation - the parser may fail when end tokens appear in content
// For now, we accept this behavior
if result.is_empty() {
if tools.is_empty() {
// Parser failed due to nested tokens - this is expected
assert_eq!(
result.len(),
tools.len(),
0,
"Known limitation: nested wrapper tokens in content"
);
} else {
// If it does parse, verify it's correct
assert_eq!(result[0].function.name, "echo");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
assert_eq!(tools[0].function.name, "echo");
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["text"], "Use <call> and </call> tags");
}
}
......@@ -118,9 +118,9 @@ async fn test_json_extraction_without_wrapper_tokens() {
And here is some text after.
"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "search");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "search");
}
#[tokio::test]
......@@ -143,9 +143,9 @@ async fn test_json_with_multiline_wrapper_content() {
```
Done!"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "format_code");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "format_code");
}
#[tokio::test]
......@@ -158,11 +158,11 @@ async fn test_json_with_special_chars_in_tokens() {
let input = r#"{{FUNC[[{"name": "test", "arguments": {"special": "[]{}"}}]]FUNC}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
let args: serde_json::Value = serde_json::from_str(&result[0].function.arguments).unwrap();
let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap();
assert_eq!(args["special"], "[]{}");
}
......@@ -183,9 +183,9 @@ async fn test_json_multiple_tools_with_wrapper() {
// Current implementation might handle this as separate calls
// Let's test that at least the first one is parsed
let result = parser.parse_complete(input).await.unwrap();
assert!(!result.is_empty(), "Should parse at least one tool");
assert_eq!(result[0].function.name, "tool1");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert!(!tools.is_empty(), "Should parse at least one tool");
assert_eq!(tools[0].function.name, "tool1");
}
#[tokio::test]
......@@ -201,10 +201,10 @@ async fn test_json_wrapper_with_array() {
{"name": "func2", "arguments": {"param": "value"}}
]</tools>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 2);
assert_eq!(result[0].function.name, "func1");
assert_eq!(result[1].function.name, "func2");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 2);
assert_eq!(tools[0].function.name, "func1");
assert_eq!(tools[1].function.name, "func2");
}
#[tokio::test]
......@@ -217,13 +217,13 @@ async fn test_json_incomplete_wrapper_tokens() {
// Missing end token
let input = r#"<tool>{"name": "test", "arguments": {}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 0, "Should not parse without closing token");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0, "Should not parse without closing token");
// Missing start token
let input = r#"{"name": "test", "arguments": {}}</tool>"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 0, "Should not parse without opening token");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 0, "Should not parse without opening token");
}
#[tokio::test]
......@@ -236,7 +236,7 @@ async fn test_json_empty_wrapper_tokens() {
let input = r#"{"name": "test", "arguments": {"key": "value"}}"#;
let result = parser.parse_complete(input).await.unwrap();
assert_eq!(result.len(), 1);
assert_eq!(result[0].function.name, "test");
let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
assert_eq!(tools.len(), 1);
assert_eq!(tools[0].function.name, "test");
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment