feat: parse normal text along with tool calls (#2709)

lib/llm/src/protocols/openai/chat_completions/aggregator.rs

@@ -166,7 +166,7 @@ impl DeltaAggregator {
         // After aggregation, inspect each choice's text for tool call syntax
         for choice in aggregator.choices.values_mut() {
             if choice.tool_calls.is_none()
-                && let Ok(tool_calls) = try_tool_call_parse_aggregate(
+                && let Ok((tool_calls, normal_text)) = try_tool_call_parse_aggregate(
                     &choice.text,
                     parsing_options.tool_call_parser.as_deref(),
                 )
@@ -184,6 +184,10 @@ impl DeltaAggregator {
                 }
                 choice.tool_calls = Some(tool_calls);
                 choice.text.clear();
+                // If normal text is not empty, update the choice text
+                if let Some(normal_text) = normal_text.filter(|text| !text.is_empty()) {
+                    choice.text = normal_text;
+                }
                 choice.finish_reason = Some(dynamo_async_openai::types::FinishReason::ToolCalls);
             }
         }
@@ -223,7 +227,7 @@ impl From<DeltaChoice> for dynamo_async_openai::types::ChatChoice {
         dynamo_async_openai::types::ChatChoice {
             message: dynamo_async_openai::types::ChatCompletionResponseMessage {
                 role: delta.role.expect("delta should have a Role"),
-                content: if delta.tool_calls.is_some() {
+                content: if delta.text.is_empty() {
                     None
                 } else {
                     Some(delta.text)
@@ -582,4 +586,68 @@ mod tests {
             dynamo_async_openai::types::Role::Assistant
         );
     }
+
+    #[tokio::test]
+    async fn test_tool_calling_output_with_normal_text() {
+        // Simulate a delta with a tool call in the content
+        let tool_call_json = r#"Hey, I'm a normal text! {"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}"#;
+
+        // Use create_test_delta to generate the annotated delta, then extract the inner delta for the test
+        let annotated_delta = create_test_delta(
+            0,
+            tool_call_json,
+            Some(dynamo_async_openai::types::Role::Assistant),
+            Some(dynamo_async_openai::types::FinishReason::ToolCalls),
+        );
+        let data = annotated_delta.data.unwrap();
+
+        // Wrap it in Annotated and create a stream
+        let annotated_delta = Annotated {
+            data: Some(data),
+            id: Some("test_id".to_string()),
+            event: None,
+            comment: None,
+        };
+        let stream = Box::pin(stream::iter(vec![annotated_delta]));
+
+        // Call DeltaAggregator::apply
+        let result = DeltaAggregator::apply(stream, ParsingOptions::default()).await;
+
+        // Check the result
+        assert!(result.is_ok());
+        let response = result.unwrap();
+
+        // There should be one choice
+        assert_eq!(response.choices.len(), 1);
+        let choice = &response.choices[0];
+
+        // The tool_calls field should be present and parsed
+        assert!(choice.message.tool_calls.is_some());
+        let tool_calls = choice.message.tool_calls.as_ref().unwrap();
+        assert_eq!(tool_calls.len(), 1);
+
+        let tool_call = &tool_calls[0];
+        assert_eq!(tool_call.function.name, "get_weather");
+        // The arguments should be a JSON string containing the expected keys
+        let args: serde_json::Value = serde_json::from_str(&tool_call.function.arguments).unwrap();
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+
+        // The content should be the normal text
+        assert!(choice.message.content.is_some());
+        assert_eq!(
+            choice.message.content.as_ref().unwrap(),
+            "Hey, I'm a normal text!"
+        );
+
+        // The finish_reason should be ToolCalls
+        assert_eq!(
+            choice.finish_reason,
+            Some(dynamo_async_openai::types::FinishReason::ToolCalls)
+        );
+        assert_eq!(
+            choice.message.role,
+            dynamo_async_openai::types::Role::Assistant
+        );
+    }
 }

lib/parsers/src/tool_calling/json_parser.rs

@@ -60,10 +60,10 @@ fn extract_tool_call_content(input: &str, start_token: &str, end_token: &str) ->
 
 // Special case for <|python_tag|> . Regex pattern does not work well with it as it has no end token
 // Handles single tool and multiple tool call cases for single start_token like <|python_tag|>
-fn handle_single_token_tool_calls(input: &str, start_token: &str) -> String {
+fn handle_single_token_tool_calls(input: &str, start_token: &str) -> Option<String> {
     // Return the input if it doesn't contain the start token
     if !input.contains(start_token) {
-        return input.to_string();
+        return None;
     }
 
     // Split on the start token and keep only JSON-looking segments
@@ -89,13 +89,23 @@ fn handle_single_token_tool_calls(input: &str, start_token: &str) -> String {
         // Remove everything up to and including the first occurrence of the start token
         if let Some(idx) = input.find(start_token) {
             let rest = &input[idx + start_token.len()..];
-            return rest.trim_start().to_string();
+            return Some(rest.trim_start().to_string());
         } else {
             // Shouldn't happen because we checked contains() above, but be defensive
-            return input.to_string();
+            return None;
         }
     }
-    format!("[{}]", items.join(","))
+    Some(format!("[{}]", items.join(",")))
+}
+
+fn try_parse_normal_text(input: &str, start_token: &str) -> String {
+    // If input contains start token, just take the part before it
+    if let Some(idx) = input.find(start_token) {
+        return input[..idx].trim().to_string();
+    }
+
+    // No start token found, return empty string
+    String::new()
 }
 
 /// Attempts to parse a tool call from a raw LLM message string into a unified [`ToolCallResponse`] format.
@@ -142,40 +152,81 @@ fn handle_single_token_tool_calls(input: &str, start_token: &str) -> String {
 pub fn try_tool_call_parse_json(
     message: &str,
     config: &JsonParserConfig,
-) -> anyhow::Result<Vec<ToolCallResponse>> {
+) -> anyhow::Result<(Vec<ToolCallResponse>, Option<String>)> {
     // Log the config we are using
     tracing::debug!("Using JSON parser config: {:?}", config);
     let trimmed = message.trim();
 
-    // Use config to get tool call start and end token vectors, then use the first element for now
+    // Early exit if no content
+    if trimmed.is_empty() {
+        return Ok((vec![], Some(String::new())));
+    }
+
     let tool_call_start_tokens = &config.tool_call_start_tokens;
     let tool_call_end_tokens = &config.tool_call_end_tokens;
 
-    assert!(
-        tool_call_start_tokens.len() == tool_call_end_tokens.len(),
-        "Tool call start and end tokens must have the same length"
-    );
+    // Early exit if no tokens configured
+    if tool_call_start_tokens.is_empty() {
+        return Ok((vec![], Some(trimmed.to_string())));
+    }
 
     // Iterate over all start and end tokens and try to extract the content between them
     // Assumption : One message will not contain different tags for tool calls. Iteration over tags is to support different tags by default for multiple models
     let mut json = trimmed.to_string();
-    for (start_token, end_token) in tool_call_start_tokens
+    let mut normal_text = trimmed.to_string();
+
+    // First, check if ANY start token exists in the input
+    let has_start_token = tool_call_start_tokens
         .iter()
-        .zip(tool_call_end_tokens.iter())
-    {
-        // Special case for <|python_tag|> . Regex pattern does not work well with it as it has no end token
-        json = if !start_token.is_empty() && end_token.is_empty() {
-            handle_single_token_tool_calls(&json, start_token)
-        } else if let Some(content) = extract_tool_call_content(&json, start_token, end_token) {
-            content
-        } else {
-            json
-        };
-    }
+        .any(|token| !token.is_empty() && normal_text.contains(token));
 
+    if !has_start_token {
+        // No start tokens found, try to extract JSON directly. Everything that starts with { or [ is considered a potential JSON.
+        if let Some(idx) = normal_text.find(['{', '[']) {
+            let extracted_normal = normal_text[..idx].trim().to_string();
+            let extracted_json = normal_text[idx..].trim().to_string();
+            if !extracted_json.is_empty() {
+                normal_text = extracted_normal;
+                json = extracted_json;
+            }
+        }
+    } else {
+        // Start tokens exist, use regex-based parsing
+        for (start_token, end_token) in tool_call_start_tokens
+            .iter()
+            .zip(tool_call_end_tokens.iter())
+        {
+            let new_normal_text = try_parse_normal_text(&normal_text, start_token);
+
+            // Process based on token types
+            match (start_token.is_empty(), end_token.is_empty()) {
+                (false, true) => {
+                    // Single token case
+                    let result = handle_single_token_tool_calls(&json, start_token);
+                    if let Some(content) = result {
+                        json = content;
+                        // For single token case, use the normal text we extracted earlier
+                        normal_text = new_normal_text;
+                        break; // Found content, exit early
+                    }
+                }
+                (false, false) => {
+                    // Start and end token case
+                    let result = extract_tool_call_content(&json, start_token, end_token);
+                    if let Some(content) = result {
+                        json = content;
+                        normal_text = new_normal_text;
+                        break; // Found content, exit early
+                    }
+                }
+                _ => {
+                    continue;
+                }
+            }
+        }
+    }
     // Convert json (String) to &str
     let json = json.as_str();
-
     // Anonymous function to attempt deserialization into a known representation
     let parse = |name: String, args: HashMap<String, Value>| -> anyhow::Result<_> {
         Ok(ToolCallResponse {
@@ -198,7 +249,10 @@ pub fn try_tool_call_parse_json(
     //   }
     // }
     if let Ok(single) = serde_json::from_str::<CalledFunctionParameters>(json) {
-        return Ok(vec![parse(single.name, single.parameters)?]);
+        return Ok((
+            vec![parse(single.name, single.parameters)?],
+            Some(normal_text),
+        ));
         //parse(single.name, single.parameters).map(Some);
 
         // CalledFunctionArguments: Single { name, arguments }
@@ -211,7 +265,10 @@ pub fn try_tool_call_parse_json(
         //   }
         // }
     } else if let Ok(single) = serde_json::from_str::<CalledFunctionArguments>(json) {
-        return Ok(vec![parse(single.name, single.arguments)?]);
+        return Ok((
+            vec![parse(single.name, single.arguments)?],
+            Some(normal_text),
+        ));
 
     // Vec<CalledFunctionParameters>: List of { name, parameters }
     // Example:
@@ -225,7 +282,7 @@ pub fn try_tool_call_parse_json(
         for item in list {
             results.push(parse(item.name, item.parameters)?);
         }
-        return Ok(results);
+        return Ok((results, Some(normal_text)));
 
     // Vec<CalledFunctionArguments>: List of { name, arguments }
     // Example:
@@ -244,8 +301,8 @@ pub fn try_tool_call_parse_json(
         for item in list {
             results.push(parse(item.name, item.arguments)?);
         }
-        return Ok(results);
+        return Ok((results, Some(normal_text)));
     }
 
-    Ok(vec![])
+    Ok((vec![], Some(trimmed.to_string())))
 }

lib/parsers/src/tool_calling/parsers.rs

@@ -135,10 +135,13 @@ pub struct ToolCallConfig {
 pub fn try_tool_call_parse(
     message: &str,
     config: &ToolCallConfig,
-) -> anyhow::Result<Vec<ToolCallResponse>> {
+) -> anyhow::Result<(Vec<ToolCallResponse>, Option<String>)> {
     // Use match statement (Rust's switch statement) to call the appropriate parser
     match config.format {
-        ToolCallParserType::Json => try_tool_call_parse_json(message, &config.json),
+        ToolCallParserType::Json => {
+            let (results, normal_content) = try_tool_call_parse_json(message, &config.json)?;
+            Ok((results, normal_content))
+        }
         ToolCallParserType::Harmony => {
             anyhow::bail!("Harmony parser not implemented");
         }
@@ -158,7 +161,7 @@ pub fn try_tool_call_parse(
 pub fn detect_and_parse_tool_call(
     message: &str,
     parser_str: Option<&str>,
-) -> anyhow::Result<Vec<ToolCallResponse>> {
+) -> anyhow::Result<(Vec<ToolCallResponse>, Option<String>)> {
     let mut parser_map: std::collections::HashMap<&str, ToolCallConfig> =
         std::collections::HashMap::new();
     parser_map.insert("hermes", ToolCallConfig::hermes());
@@ -175,7 +178,10 @@ pub fn detect_and_parse_tool_call(
     };
 
     match parser_map.get(parser_key) {
-        Some(config) => try_tool_call_parse(message, config),
+        Some(config) => {
+            let (results, normal_content) = try_tool_call_parse(message, config)?;
+            Ok((results, normal_content))
+        }
         None => anyhow::bail!("Parser for the given config is not implemented"), // Original message
     }
 }
@@ -194,7 +200,8 @@ mod tests {
     #[test]
     fn parses_single_parameters_object() {
         let input = r#"{ "name": "hello", "parameters": { "x": 1, "y": 2 } }"#;
-        let result = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -206,7 +213,8 @@ mod tests {
     #[test]
     fn parses_single_arguments_object() {
         let input = r#"{ "name": "world", "arguments": { "a": "abc", "b": 42 } }"#;
-        let result = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -218,7 +226,8 @@ mod tests {
     #[test]
     fn parses_vec_of_parameters() {
         let input = r#"[{ "name": "first", "parameters": { "a": 1 } }, { "name": "second", "parameters": { "b": 2 } }]"#;
-        let result = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -232,7 +241,8 @@ mod tests {
     #[test]
     fn parses_vec_of_arguments() {
         let input = r#"[{ "name": "alpha", "arguments": { "a": "x" } }, { "name": "omega", "arguments": { "z": "y" } }]"#;
-        let result = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -247,7 +257,8 @@ mod tests {
     fn parses_toolcall_wrapped_payload() {
         let input =
             r#"<TOOLCALL>[{ "name": "wrapped", "parameters": { "foo": "bar" } }]</TOOLCALL>"#;
-        let result = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -258,7 +269,7 @@ mod tests {
     #[test]
     fn parses_python_tag_prefixed_payload() {
         let input = r#"<|python_tag|>{ "name": "pyfunc", "arguments": { "k": "v" } }"#;
-        let result = try_tool_call_parse(
+        let (result, content) = try_tool_call_parse(
             input,
             &ToolCallConfig {
                 format: ToolCallParserType::Json,
@@ -270,6 +281,7 @@ mod tests {
             },
         )
         .unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -280,14 +292,16 @@ mod tests {
     #[test]
     fn returns_none_on_invalid_input() {
         let input = r#"not even json"#;
-        let result = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        assert_eq!(content, Some("not even json".to_string()));
         assert!(result.is_empty());
     }
 
     #[test]
     fn returns_none_on_valid_json_wrong_shape() {
         let input = r#"{ "foo": "bar" }"#;
-        let result = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        assert_eq!(content, Some("{ \"foo\": \"bar\" }".to_string()));
         assert!(result.is_empty());
     }
 
@@ -299,9 +313,23 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
 </think>
 
 <TOOLCALL>[{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}]</TOOLCALL>"#;
-        let result = detect_and_parse_tool_call(input, Some("nemotron_deci")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("nemotron_deci")).unwrap();
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
+        assert_eq!(content, Some("<think>\nOkay, the user is asking for the weather in San Francisco in Fahrenheit. Let me check the tools available.\n</think>".to_string()));
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_nvidia_llama3_nemotron_super_49b_simple_with_no_think() {
+        let input = r#"<TOOLCALL>[{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}]</TOOLCALL>"#;
+        let (result, content) = detect_and_parse_tool_call(input, Some("nemotron_deci")).unwrap();
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 1);
+        assert_eq!(content, Some("".to_string()));
         let (name, args) = extract_name_and_args(result[0].clone());
         assert_eq!(name, "get_weather");
         assert_eq!(args["location"], "San Francisco, CA");
@@ -316,7 +344,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
 
 <TOOLCALL>[{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "New York, NY", "unit": "fahrenheit"}}]</TOOLCALL>"#;
         let config = ToolCallConfig::nemotron_deci();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("<think>\nOkay, the user is asking for the weather in San Francisco in Fahrenheit. Let me check the tools available.\n</think>".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -346,7 +375,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
   </TOOLCALL>
   "#;
         let config = ToolCallConfig::nemotron_deci();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("<think>\nOkay, the user is asking for the weather in San Francisco in Fahrenheit. Let me check the tools available.\n</think>".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -364,7 +394,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
         let input = r#"<tool_call>
 {"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}
 </tool_call>"#;
-        let result = detect_and_parse_tool_call(input, Some("hermes")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("hermes")).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -373,12 +404,24 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
         assert_eq!(args["unit"], "fahrenheit");
     }
 
+    #[test]
+    fn test_qwen_qwq_32b_simple_with_normal_text() {
+        let input = r#"Hey How are you? <tool_call>
+{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}
+</tool_call>"#;
+        let (result, content) = detect_and_parse_tool_call(input, Some("hermes")).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 1);
+    }
+
     #[test]
     fn test_nousresearch_hermes3_llama31_8b_simple() {
         let input = r#"<tool_call>
 {"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}
 </tool_call>"#;
-        let result = detect_and_parse_tool_call(input, Some("hermes")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("hermes")).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -397,7 +440,32 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
 </tool_call>
 "#;
         let config = ToolCallConfig::hermes();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 2);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+        let (name, args) = extract_name_and_args(result[1].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "New York, NY");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_qwen_qwq_32b_multiple_tool_calls_with_normal_text() {
+        let input = r#"Hey How are you? <tool_call>
+{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}
+</tool_call>
+<tool_call>
+{"name": "get_weather", "arguments": {"location": "New York, NY", "unit": "fahrenheit"}}
+</tool_call>
+"#;
+        let config = ToolCallConfig::hermes();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -424,7 +492,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
 </tool_call>
 "#;
         let config = ToolCallConfig::hermes();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -450,7 +519,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
                 ..Default::default()
             },
         };
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -462,16 +532,23 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
     #[test]
     fn test_mistralai_mistral_7b_instruct_v03_simple() {
         let input = r#" [{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}]"#;
-        let config = ToolCallConfig {
-            format: ToolCallParserType::Json,
-            json: JsonParserConfig {
-                tool_call_start_tokens: vec![],
-                tool_call_end_tokens: vec![],
-                arguments_keys: vec!["arguments".to_string()],
-                ..Default::default()
-            },
-        };
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let config = ToolCallConfig::mistral();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 1);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_mistralai_mistral_7b_instruct_v03_simple_with_normal_text() {
+        let input = r#"Hey How are you? [{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}]"#;
+        let config = ToolCallConfig::mistral();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -488,16 +565,9 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
         "San Francisco, CA",
         "unit": "fahrenheit"}}]
         "#;
-        let config = ToolCallConfig {
-            format: ToolCallParserType::Json,
-            json: JsonParserConfig {
-                tool_call_start_tokens: vec![],
-                tool_call_end_tokens: vec![],
-                arguments_keys: vec!["arguments".to_string()],
-                ..Default::default()
-            },
-        };
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let config = ToolCallConfig::mistral();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -510,7 +580,26 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
     fn test_mistralai_mistral_7b_instruct_v03_multiple() {
         let input = r#" [{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "New York, NY", "unit": "fahrenheit"}}]"#;
         let config = ToolCallConfig::mistral();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 2);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+        let (name, args) = extract_name_and_args(result[1].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "New York, NY");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_mistralai_mistral_7b_instruct_v03_multiple_with_normal_text() {
+        let input = r#"Hey How are you? [{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "New York, NY", "unit": "fahrenheit"}}]"#;
+        let config = ToolCallConfig::mistral();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -527,12 +616,16 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
     fn test_mistralai_mistral_7b_instruct_v03_multiple_with_new_lines() {
         let input = r#"
         [{"name": "get_weather",
-        "arguments": {"location": "San Francisco, CA",
-        "unit": "fahrenheit"}}, {"name": "get_weather", "arguments":
-        {"location": "New York, NY", "unit": "fahrenheit"}}]
+        "arguments": {"location":
+        "San Francisco, CA",
+        "unit": "fahrenheit"}},
+        {"name": "get_weather", "arguments":
+        {"location": "New York, NY", "unit":
+        "fahrenheit"}}]
         "#;
         let config = ToolCallConfig::mistral();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -549,7 +642,22 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
     fn test_mistralai_mistral_7b_instruct_v03_single_with_start_token() {
         let input = r#"[TOOL_CALLS] [{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}]"#;
         let config = ToolCallConfig::mistral();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 1);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_mistralai_mistral_7b_instruct_v03_single_with_start_token_with_normal_text() {
+        let input = r#"Hey How are you? [TOOL_CALLS] [{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}]"#;
+        let config = ToolCallConfig::mistral();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -568,7 +676,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
         "unit": "fahrenheit"}}]
         "#;
         let config = ToolCallConfig::mistral();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -581,7 +690,26 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
     fn test_mistralai_mistral_7b_instruct_v03_single_with_start_token_multiple() {
         let input = r#"[TOOL_CALLS] [{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "New York, NY", "unit": "fahrenheit"}}]"#;
         let config = ToolCallConfig::mistral();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 2);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+        let (name, args) = extract_name_and_args(result[1].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "New York, NY");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_mistralai_mistral_7b_instruct_v03_single_with_start_token_multiple_with_normal_text() {
+        let input = r#"Hey How are you? [TOOL_CALLS] [{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "New York, NY", "unit": "fahrenheit"}}]"#;
+        let config = ToolCallConfig::mistral();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -607,7 +735,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
         "fahrenheit"}}]
         "#;
         let config = ToolCallConfig::mistral();
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -623,7 +752,21 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
     #[test]
     fn test_meta_llama_llama31_8b_instruct_simple() {
         let input = r#"{"name": "get_weather", "parameters": {"location": "San Francisco, CA", "unit": "fahrenheit"}}"#;
-        let result = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::mistral()).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 1);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_meta_llama_llama31_8b_instruct_simple_with_normal_text() {
+        let input = r#"Hey How are you? {"name": "get_weather", "parameters": {"location": "San Francisco, CA", "unit": "fahrenheit"}}"#;
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::mistral()).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -638,7 +781,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
         {"name": "get_weather",
         "parameters": {"location": "San Francisco, CA", "unit": "fahrenheit"}}
         "#;
-        let result = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -650,7 +794,21 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
     #[test]
     fn test_meta_llama_llama31_8b_instruct_with_python_tag() {
         let input = r#"<|python_tag|>{ "name": "get_weather", "parameters": {"location": "San Francisco, CA", "unit": "fahrenheit" } }"#;
-        let result = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 1);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_meta_llama_llama31_8b_instruct_with_python_tag_with_normal_text() {
+        let input = r#"Hey How are you? <|python_tag|>{ "name": "get_weather", "parameters": {"location": "San Francisco, CA", "unit": "fahrenheit" } }"#;
+        let (result, content) = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -665,7 +823,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
         <|python_tag|>
         {"name": "get_weather", "parameters": {"location": "San Francisco, CA", "unit": "fahrenheit"}}
         "#;
-        let result = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -682,7 +841,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
         <|python_tag|>
         {"name": "get_weather", "parameters": {"location": "New York, NY", "unit": "fahrenheit" }}
         "#;
-        let result = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("llama3_json")).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -710,15 +870,15 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
 
         // Known parser, but invalid input (not JSON) should return Ok(None)
         let input = "not a json";
-        let result = detect_and_parse_tool_call(input, Some("hermes"));
-        assert!(result.is_ok());
-        assert!(result.unwrap().is_empty());
+        let (result, content) = detect_and_parse_tool_call(input, Some("hermes")).unwrap();
+        assert_eq!(content, Some("not a json".to_string()));
+        assert!(result.is_empty());
 
         // Known parser, but valid JSON with wrong shape should return Ok(None)
         let input = r#"{"foo": "bar"}"#;
-        let result = detect_and_parse_tool_call(input, Some("hermes"));
-        assert!(result.is_ok());
-        assert!(result.unwrap().is_empty());
+        let (result, content) = detect_and_parse_tool_call(input, Some("hermes")).unwrap();
+        assert_eq!(content, Some(r#"{"foo": "bar"}"#.to_string()));
+        assert!(result.is_empty());
     }
 
     #[test]
@@ -729,7 +889,8 @@ Okay, the user is asking for the weather in San Francisco in Fahrenheit. Let me
 - **Summer (June to August)**: Average highs range from the mid-60s to low 70s Fahrenheit, with cooler mornings and evenings. Coastal areas may be cooler than inland spots.
 
 Remember, San Francisco weather can be quite unpredictable, particularly with its famous fog, which can significantly lower temperatures. Always check a local weather forecast for the most accurate and up-to-date information."#;
-        let result = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::default()).unwrap();
+        assert_eq!(content, Some(input.to_string()));
         assert!(result.is_empty()); // This model doesn't produce tool calls
     }
 
@@ -748,7 +909,8 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
                 ..Default::default()
             },
         };
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -770,7 +932,8 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
                 ..Default::default()
             },
         };
-        let result = try_tool_call_parse(input, &config).unwrap();
+        let (result, content) = try_tool_call_parse(input, &config).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -782,7 +945,8 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
     #[test]
     fn test_detect_and_parse_tool_call_default_parser_nemotron_deci() {
         let input = r#"<TOOLCALL>[{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}]</TOOLCALL>"#;
-        let result = detect_and_parse_tool_call(input, None).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, None).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -794,7 +958,25 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
     #[test]
     fn test_detect_and_parse_tool_call_default_parser_nemotron_deci_multiple() {
         let input = r#"<TOOLCALL>[{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "New York, NY", "unit": "fahrenheit"}}]</TOOLCALL>"#;
-        let result = detect_and_parse_tool_call(input, None).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, None).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 2);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+        let (name, args) = extract_name_and_args(result[1].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "New York, NY");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_detect_and_parse_tool_call_default_parser_nemotron_deci_multiple_with_normal_text() {
+        let input = r#"Hey How are you? <TOOLCALL>[{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit"}}, {"name": "get_weather", "arguments": {"location": "New York, NY", "unit": "fahrenheit"}}]</TOOLCALL>"#;
+        let (result, content) = detect_and_parse_tool_call(input, None).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 2);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -810,7 +992,22 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
     #[test]
     fn test_detect_and_parse_tool_call_default_parser_llama3_json_with_python_tag() {
         let input = r#"<|python_tag|>{ "name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit" } }"#;
-        let result = detect_and_parse_tool_call(input, None).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, None).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 1);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_detect_and_parse_tool_call_default_parser_llama3_json_with_python_tag_with_normal_text()
+    {
+        let input = r#"Hey How are you? <|python_tag|>{ "name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit" } }"#;
+        let (result, content) = detect_and_parse_tool_call(input, None).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -829,7 +1026,8 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
           {"location": "San Francisco, CA",
           "unit": "fahrenheit" }}
         "#;
-        let result = detect_and_parse_tool_call(input, None).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, None).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -846,7 +1044,8 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
          {"location": "San Francisco, CA",
           "unit": "fahrenheit" }}
         "#;
-        let result = detect_and_parse_tool_call(input, None).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, None).unwrap();
+        assert_eq!(content, Some("".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -858,7 +1057,22 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
     #[test]
     fn test_detect_and_parse_tool_call_default_parser_llama3_json_without_python_tag() {
         let input = r#"{ "name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit" } }"#;
-        let result = detect_and_parse_tool_call(input, None).unwrap();
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::mistral()).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert!(!result.is_empty());
+        assert_eq!(result.len(), 1);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather");
+        assert_eq!(args["location"], "San Francisco, CA");
+        assert_eq!(args["unit"], "fahrenheit");
+    }
+
+    #[test]
+    fn test_detect_and_parse_tool_call_default_parser_llama3_json_without_python_tag_with_normal_text()
+     {
+        let input = r#"Hey How are you? { "name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "fahrenheit" } }"#;
+        let (result, content) = try_tool_call_parse(input, &ToolCallConfig::mistral()).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert!(!result.is_empty());
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
@@ -871,7 +1085,19 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
     fn test_phi4_single_function_call() {
         let input =
             r#"functools[{"name": "get_country_capital", "arguments": {"country": "Poland"}}]"#;
-        let result = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert_eq!(result.len(), 1);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_country_capital");
+        assert_eq!(args["country"], "Poland");
+    }
+
+    #[test]
+    fn test_phi4_single_function_call_with_normal_text() {
+        let input = r#"Hey How are you? functools[{"name": "get_country_capital", "arguments": {"country": "Poland"}}]"#;
+        let (result, content) = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
         assert_eq!(name, "get_country_capital");
@@ -884,7 +1110,27 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
   {"name": "get_country_capital", "arguments": {"country": "Poland"}},
   {"name": "get_population", "arguments": {"city": "Warsaw"}}
 ]"#;
-        let result = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert_eq!(result.len(), 2);
+
+        let (name1, args1) = extract_name_and_args(result[0].clone());
+        assert_eq!(name1, "get_country_capital");
+        assert_eq!(args1["country"], "Poland");
+
+        let (name2, args2) = extract_name_and_args(result[1].clone());
+        assert_eq!(name2, "get_population");
+        assert_eq!(args2["city"], "Warsaw");
+    }
+
+    #[test]
+    fn test_phi4_multiple_function_calls_simple_arguments_with_normal_text() {
+        let input = r#"Hey How are you? functools[
+  {"name": "get_country_capital", "arguments": {"country": "Poland"}},
+  {"name": "get_population", "arguments": {"city": "Warsaw"}}
+]"#;
+        let (result, content) = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert_eq!(result.len(), 2);
 
         let (name1, args1) = extract_name_and_args(result[0].clone());
@@ -901,7 +1147,23 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
         let input = r#"functools[{"name": "get_weather_forecast", "arguments":
         {"location": {"city": "San Francisco",
         "state": "CA"}, "date": "2023-10-05"}}]"#;
-        let result = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert_eq!(result.len(), 1);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "get_weather_forecast");
+        assert_eq!(args["date"], "2023-10-05");
+        assert_eq!(args["location"]["city"], "San Francisco");
+        assert_eq!(args["location"]["state"], "CA");
+    }
+
+    #[test]
+    fn test_phi4_single_function_call_nested_json_arguments_with_normal_text() {
+        let input = r#"Hey How are you? functools[{"name": "get_weather_forecast", "arguments":
+        {"location": {"city": "San Francisco",
+        "state": "CA"}, "date": "2023-10-05"}}]"#;
+        let (result, content) = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
         assert_eq!(name, "get_weather_forecast");
@@ -914,7 +1176,21 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it
     fn test_phi4_function_call_with_parameters_instead_of_arguments() {
         let input = r#"functools[{"name": "calculate_distance",
          "parameters": {"from": "New York", "to": "Los Angeles"}}]"#;
-        let result = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        let (result, content) = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        assert_eq!(content, Some("".to_string()));
+        assert_eq!(result.len(), 1);
+        let (name, args) = extract_name_and_args(result[0].clone());
+        assert_eq!(name, "calculate_distance");
+        assert_eq!(args["from"], "New York");
+        assert_eq!(args["to"], "Los Angeles");
+    }
+
+    #[test]
+    fn test_phi4_function_call_with_parameters_instead_of_arguments_with_normal_text() {
+        let input = r#"Hey How are you? functools[{"name": "calculate_distance",
+         "parameters": {"from": "New York", "to": "Los Angeles"}}]"#;
+        let (result, content) = detect_and_parse_tool_call(input, Some("phi4")).unwrap();
+        assert_eq!(content, Some("Hey How are you?".to_string()));
         assert_eq!(result.len(), 1);
         let (name, args) = extract_name_and_args(result[0].clone());
         assert_eq!(name, "calculate_distance");

lib/parsers/src/tool_calling/tools.rs

@@ -13,29 +13,35 @@ pub use super::parsers::{ToolCallConfig, detect_and_parse_tool_call};
 pub fn try_tool_call_parse_aggregate(
     message: &str,
     parser_str: Option<&str>,
-) -> anyhow::Result<Vec<dynamo_async_openai::types::ChatCompletionMessageToolCall>> {
+) -> anyhow::Result<(
+    Vec<dynamo_async_openai::types::ChatCompletionMessageToolCall>,
+    Option<String>,
+)> {
     if parser_str.is_none() {
         tracing::info!("No tool parser provided. Trying parsing with default parser.");
     } else {
         tracing::info!("Using tool parser: {:?}", parser_str);
     }
-    let parsed = detect_and_parse_tool_call(message, parser_str)?;
+    let (parsed, content) = detect_and_parse_tool_call(message, parser_str)?;
     if parsed.is_empty() {
-        return Ok(vec![]);
+        return Ok((vec![], content));
     }
-    Ok(parsed
-        .into_iter()
-        .map(
-            |parsed| dynamo_async_openai::types::ChatCompletionMessageToolCall {
-                id: parsed.id,
-                r#type: dynamo_async_openai::types::ChatCompletionToolType::Function,
-                function: dynamo_async_openai::types::FunctionCall {
-                    name: parsed.function.name,
-                    arguments: parsed.function.arguments,
+    Ok((
+        parsed
+            .into_iter()
+            .map(
+                |parsed| dynamo_async_openai::types::ChatCompletionMessageToolCall {
+                    id: parsed.id,
+                    r#type: dynamo_async_openai::types::ChatCompletionToolType::Function,
+                    function: dynamo_async_openai::types::FunctionCall {
+                        name: parsed.function.name,
+                        arguments: parsed.function.arguments,
+                    },
                 },
-            },
-        )
-        .collect())
+            )
+            .collect(),
+        content,
+    ))
 }
 
 /// Try parsing a string as a structured tool call, for streaming (delta) usage.
@@ -44,25 +50,31 @@ pub fn try_tool_call_parse_aggregate(
 pub fn try_tool_call_parse_stream(
     message: &str,
     parser_str: Option<&str>,
-) -> anyhow::Result<Vec<dynamo_async_openai::types::ChatCompletionMessageToolCallChunk>> {
-    let parsed = detect_and_parse_tool_call(message, parser_str)?;
+) -> anyhow::Result<(
+    Vec<dynamo_async_openai::types::ChatCompletionMessageToolCallChunk>,
+    Option<String>,
+)> {
+    let (parsed, content) = detect_and_parse_tool_call(message, parser_str)?;
     if parsed.is_empty() {
-        return Ok(vec![]);
+        return Ok((vec![], content));
     }
-    Ok(parsed
-        .into_iter()
-        .enumerate()
-        .map(
-            |(idx, parsed)| dynamo_async_openai::types::ChatCompletionMessageToolCallChunk {
-                index: idx as u32,
-                id: Some(parsed.id),
-                r#type: Some(dynamo_async_openai::types::ChatCompletionToolType::Function),
-                function: Some(dynamo_async_openai::types::FunctionCallStream {
-                    name: Some(parsed.function.name),
-                    arguments: Some(parsed.function.arguments),
-                }),
-                // Add other fields as needed if required by the struct definition
-            },
-        )
-        .collect())
+    Ok((
+        parsed
+            .into_iter()
+            .enumerate()
+            .map(
+                |(idx, parsed)| dynamo_async_openai::types::ChatCompletionMessageToolCallChunk {
+                    index: idx as u32,
+                    id: Some(parsed.id),
+                    r#type: Some(dynamo_async_openai::types::ChatCompletionToolType::Function),
+                    function: Some(dynamo_async_openai::types::FunctionCallStream {
+                        name: Some(parsed.function.name),
+                        arguments: Some(parsed.function.arguments),
+                    }),
+                    // Add other fields as needed if required by the struct definition
+                },
+            )
+            .collect(),
+        content,
+    ))
 }