//! Step3 Parser Integration Tests use sglang_router_rs::tool_parser::{Step3Parser, ToolParser}; mod common; use common::create_test_tools; #[tokio::test] async fn test_step3_complete_parsing() { let parser = Step3Parser::new(); let input = r#"Let me help you. <|tool_calls_begin|> <|tool_call_begin|>function<|tool_sep|> rust programming 10 <|tool_call_end|> <|tool_calls_end|> Here are the results..."#; let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); assert_eq!(tools.len(), 1); assert_eq!(normal_text, "Let me help you.\n"); assert_eq!(tools[0].function.name, "search"); let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); assert_eq!(args["query"], "rust programming"); assert_eq!(args["limit"], 10); } #[tokio::test] async fn test_step3_multiple_tools() { let parser = Step3Parser::new(); let input = r#"<|tool_calls_begin|> <|tool_call_begin|>function<|tool_sep|> Tokyo <|tool_call_end|> <|tool_call_begin|>function<|tool_sep|> tech 5 <|tool_call_end|> <|tool_calls_end|>"#; let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); assert_eq!(tools.len(), 2); assert_eq!(tools[0].function.name, "get_weather"); assert_eq!(tools[1].function.name, "get_news"); } #[tokio::test] async fn test_step3_type_conversion() { let parser = Step3Parser::new(); let input = r#"<|tool_calls_begin|> <|tool_call_begin|>function<|tool_sep|> 100 2.5 true null hello world <|tool_call_end|> <|tool_calls_end|>"#; let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); assert_eq!(tools.len(), 1); let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); assert_eq!(args["count"], 100); assert_eq!(args["rate"], 2.5); assert_eq!(args["active"], true); assert_eq!(args["optional"], serde_json::Value::Null); assert_eq!(args["text"], "hello world"); } #[tokio::test] async fn test_step3_streaming() { let mut parser = Step3Parser::new(); let tools = create_test_tools(); // Simulate streaming chunks let chunks = vec![ "<|tool_calls_begin|>\n", "<|tool_call_begin|>function", "<|tool_sep|>", "\n10", "\n20", "\n<|tool_call_end|>", "\n<|tool_calls_end|>", ]; let mut found_complete = false; for chunk in chunks { let result = parser.parse_incremental(chunk, &tools).await.unwrap(); if !result.calls.is_empty() { if let Some(name) = &result.calls[0].name { assert_eq!(name, "calc"); found_complete = true; } } } assert!(found_complete); } #[test] fn test_step3_format_detection() { let parser = Step3Parser::new(); // Should detect Step3 format assert!(parser.has_tool_markers("<|tool_calls_begin|>")); assert!(parser.has_tool_markers("text with <|tool_calls_begin|> marker")); // Should not detect other formats assert!(!parser.has_tool_markers("[TOOL_CALLS]")); assert!(!parser.has_tool_markers("")); assert!(!parser.has_tool_markers("plain text")); } #[tokio::test] async fn test_step3_nested_steptml() { let parser = Step3Parser::new(); let input = r#"<|tool_calls_begin|> <|tool_call_begin|>function<|tool_sep|> {"nested": {"key": "value"}} [1, 2, 3] <|tool_call_end|> <|tool_calls_end|>"#; let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); assert_eq!(tools.len(), 1); assert_eq!(tools[0].function.name, "config"); let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); assert!(args["settings"].is_object()); assert!(args["array"].is_array()); } #[tokio::test] async fn test_step3_python_literals() { let parser = Step3Parser::new(); let input = r#"<|tool_calls_begin|> <|tool_call_begin|>function<|tool_sep|> True False None <|tool_call_end|> <|tool_calls_end|>"#; let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); assert_eq!(tools.len(), 1); let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); assert_eq!(args["bool_true"], true); assert_eq!(args["bool_false"], false); assert_eq!(args["none_value"], serde_json::Value::Null); } #[tokio::test] async fn test_steptml_format() { let parser = Step3Parser::new(); let input = r#"Text before. <|tool_calls_begin|> <|tool_call_begin|>function<|tool_sep|> rust lang 10 <|tool_call_end|> <|tool_calls_end|>Text after."#; let (normal_text, tools) = parser.parse_complete(input).await.unwrap(); assert_eq!(tools.len(), 1); assert_eq!(normal_text, "Text before.\n"); assert_eq!(tools[0].function.name, "search"); let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); assert_eq!(args["query"], "rust lang"); assert_eq!(args["limit"], 10); // TODO: Verify normal text extraction } #[tokio::test] async fn test_json_parameter_values() { let parser = Step3Parser::new(); let input = r#"<|tool_calls_begin|> <|tool_call_begin|>function<|tool_sep|> {"nested": {"value": true}} [1, 2, 3] <|tool_call_end|> <|tool_calls_end|>"#; let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); assert_eq!(tools.len(), 1); let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); assert!(args["settings"].is_object()); assert!(args["items"].is_array()); } #[tokio::test] async fn test_step3_parameter_with_angle_brackets() { let parser = Step3Parser::new(); let input = r#"<|tool_calls_begin|> <|tool_call_begin|>function<|tool_sep|> a < b && b > c comparison test <|tool_call_end|> <|tool_calls_end|>"#; let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); assert_eq!(tools.len(), 1); assert_eq!(tools[0].function.name, "compare"); let args: serde_json::Value = serde_json::from_str(&tools[0].function.arguments).unwrap(); assert_eq!(args["expression"], "a < b && b > c"); assert_eq!(args["context"], "comparison test"); } #[tokio::test] async fn test_step3_empty_function_name() { let parser = Step3Parser::new(); let input = r#"<|tool_calls_begin|> <|tool_call_begin|>function<|tool_sep|> value <|tool_call_end|> <|tool_calls_end|>"#; let (_normal_text, tools) = parser.parse_complete(input).await.unwrap(); assert_eq!(tools.len(), 0); // Should reject empty function name }