tests.rs 29 KB
Newer Older
1
use super::*;
2
use crate::tool_parser::parsers::JsonParser;
3
4
5
use crate::tool_parser::partial_json::{
    compute_diff, find_common_prefix, is_complete_json, PartialJson,
};
6
use crate::tool_parser::traits::ToolParser;
7
use crate::tool_parser::types::TokenConfig;
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

#[test]
fn test_parse_state_new() {
    let state = ParseState::new();
    assert_eq!(state.phase, ParsePhase::Searching);
    assert_eq!(state.buffer, "");
    assert_eq!(state.consumed, 0);
    assert_eq!(state.bracket_depth, 0);
    assert!(!state.in_string);
    assert!(!state.escape_next);
}

#[test]
fn test_parse_state_process_char() {
    let mut state = ParseState::new();

    state.process_char('{');
    assert_eq!(state.bracket_depth, 1);

    state.process_char('}');
    assert_eq!(state.bracket_depth, 0);

    state.process_char('"');
    assert!(state.in_string);

    state.process_char('"');
    assert!(!state.in_string);

    state.process_char('"');
    state.process_char('\\');
    assert!(state.escape_next);

    state.process_char('"');
    assert!(!state.escape_next);
    assert!(state.in_string); // Still in string because quote was escaped
}

#[test]
fn test_token_config() {
    let config = TokenConfig {
        start_tokens: vec!["<start>".to_string(), "[".to_string()],
        end_tokens: vec!["</end>".to_string(), "]".to_string()],
        separator: ", ".to_string(),
    };

    let pairs: Vec<_> = config.iter_pairs().collect();
    assert_eq!(pairs.len(), 2);
    assert_eq!(pairs[0], ("<start>", "</end>"));
    assert_eq!(pairs[1], ("[", "]"));
}

#[test]
fn test_parser_registry() {
    let registry = ParserRegistry::new();

    assert!(!registry.list_mappings().is_empty());

    let mappings = registry.list_mappings();
    let has_gpt = mappings.iter().any(|(m, _)| m.starts_with("gpt"));
    assert!(has_gpt);
}

#[test]
fn test_parser_registry_pattern_matching() {
72
    let mut registry = ParserRegistry::new_for_testing();
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238

    registry.map_model("test-model", "json");

    let mappings = registry.list_mappings();
    let has_test = mappings
        .iter()
        .any(|(m, p)| *m == "test-model" && *p == "json");
    assert!(has_test);
}

#[test]
fn test_tool_call_serialization() {
    let tool_call = ToolCall {
        id: "call-123".to_string(),
        r#type: "function".to_string(),
        function: FunctionCall {
            name: "search".to_string(),
            arguments: r#"{"query": "rust programming"}"#.to_string(),
        },
    };

    let json = serde_json::to_string(&tool_call).unwrap();
    assert!(json.contains("call-123"));
    assert!(json.contains("search"));
    assert!(json.contains("rust programming"));

    let parsed: ToolCall = serde_json::from_str(&json).unwrap();
    assert_eq!(parsed.id, "call-123");
    assert_eq!(parsed.function.name, "search");
}

#[test]
fn test_partial_json_parser() {
    let parser = PartialJson::default();

    let input = r#"{"name": "test", "value": 42}"#;
    let (value, consumed) = parser.parse_value(input).unwrap();
    assert_eq!(value["name"], "test");
    assert_eq!(value["value"], 42);
    assert_eq!(consumed, input.len());

    let input = r#"{"name": "test", "value": "#;
    let (value, _consumed) = parser.parse_value(input).unwrap();
    assert_eq!(value["name"], "test");
    assert!(value["value"].is_null());

    let input = r#"{"name": "tes"#;
    let (value, _consumed) = parser.parse_value(input).unwrap();
    assert_eq!(value["name"], "tes");

    let input = r#"[1, 2, "#;
    let (value, _consumed) = parser.parse_value(input).unwrap();
    assert!(value.is_array());
    assert_eq!(value[0], 1);
    assert_eq!(value[1], 2);
}

#[test]
fn test_partial_json_depth_limit() {
    // max_depth of 3 allows nesting up to 3 levels
    // Set allow_incomplete to false to get errors instead of partial results
    let parser = PartialJson::new(3, false);

    // This should work (simple object)
    let input = r#"{"a": 1}"#;
    let result = parser.parse_value(input);
    assert!(result.is_ok());

    // This should work (nested to depth 3)
    let input = r#"{"a": {"b": {"c": 1}}}"#;
    let result = parser.parse_value(input);
    assert!(result.is_ok());

    // This should fail (nested to depth 4, exceeds limit)
    let input = r#"{"a": {"b": {"c": {"d": 1}}}}"#;
    let result = parser.parse_value(input);
    assert!(result.is_err());
}

#[test]
fn test_is_complete_json() {
    assert!(is_complete_json(r#"{"name": "test"}"#));
    assert!(is_complete_json(r#"[1, 2, 3]"#));
    assert!(is_complete_json(r#""string""#));
    assert!(is_complete_json("42"));
    assert!(is_complete_json("true"));
    assert!(is_complete_json("null"));

    assert!(!is_complete_json(r#"{"name": "#));
    assert!(!is_complete_json(r#"[1, 2, "#));
    assert!(!is_complete_json(r#""unclosed"#));
}

#[test]
fn test_find_common_prefix() {
    assert_eq!(find_common_prefix("hello", "hello"), 5);
    assert_eq!(find_common_prefix("hello", "help"), 3);
    assert_eq!(find_common_prefix("hello", "world"), 0);
    assert_eq!(find_common_prefix("", "hello"), 0);
    assert_eq!(find_common_prefix("hello", ""), 0);
}

#[test]
fn test_compute_diff() {
    assert_eq!(compute_diff("hello", "hello world"), " world");
    assert_eq!(compute_diff("", "hello"), "hello");
    assert_eq!(compute_diff("hello", "hello"), "");
    assert_eq!(compute_diff("test", "hello"), "hello");
}

#[test]
fn test_stream_result_variants() {
    let result = StreamResult::Incomplete;
    matches!(result, StreamResult::Incomplete);

    let result = StreamResult::ToolName {
        index: 0,
        name: "test".to_string(),
    };
    if let StreamResult::ToolName { index, name } = result {
        assert_eq!(index, 0);
        assert_eq!(name, "test");
    } else {
        panic!("Expected ToolName variant");
    }

    let tool = ToolCall {
        id: "123".to_string(),
        r#type: "function".to_string(),
        function: FunctionCall {
            name: "test".to_string(),
            arguments: "{}".to_string(),
        },
    };
    let result = StreamResult::ToolComplete(tool.clone());
    if let StreamResult::ToolComplete(t) = result {
        assert_eq!(t.id, "123");
    } else {
        panic!("Expected ToolComplete variant");
    }
}

#[test]
fn test_partial_tool_call() {
    let mut partial = PartialToolCall {
        name: None,
        arguments_buffer: String::new(),
        start_position: 0,
        name_sent: false,
        streamed_args: String::new(),
    };

    // Set name
    partial.name = Some("test_function".to_string());
    assert_eq!(partial.name.as_ref().unwrap(), "test_function");

    // Append arguments
    partial.arguments_buffer.push_str(r#"{"key": "value"}"#);
    assert_eq!(partial.arguments_buffer, r#"{"key": "value"}"#);

    // Update streaming state
    partial.name_sent = true;
    partial.streamed_args = r#"{"key": "#.to_string();
    assert!(partial.name_sent);
    assert_eq!(partial.streamed_args, r#"{"key": "#);
}
239
240
241
242
243
244

#[tokio::test]
async fn test_json_parser_complete_single() {
    let parser = JsonParser::new();

    let input = r#"{"name": "get_weather", "arguments": {"location": "San Francisco", "units": "celsius"}}"#;
245
    let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
246

247
248
249
250
    assert_eq!(tools.len(), 1);
    assert_eq!(tools[0].function.name, "get_weather");
    assert!(tools[0].function.arguments.contains("San Francisco"));
    assert!(tools[0].function.arguments.contains("celsius"));
251
252
253
254
255
256
257
258
259
260
261
}

#[tokio::test]
async fn test_json_parser_complete_array() {
    let parser = JsonParser::new();

    let input = r#"[
        {"name": "get_weather", "arguments": {"location": "SF"}},
        {"name": "get_news", "arguments": {"query": "technology"}}
    ]"#;

262
    let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
263

264
265
266
    assert_eq!(tools.len(), 2);
    assert_eq!(tools[0].function.name, "get_weather");
    assert_eq!(tools[1].function.name, "get_news");
267
268
269
270
271
272
273
}

#[tokio::test]
async fn test_json_parser_with_parameters() {
    let parser = JsonParser::new();

    let input = r#"{"name": "calculate", "parameters": {"x": 10, "y": 20, "operation": "add"}}"#;
274
    let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
275

276
277
278
279
280
    assert_eq!(tools.len(), 1);
    assert_eq!(tools[0].function.name, "calculate");
    assert!(tools[0].function.arguments.contains("10"));
    assert!(tools[0].function.arguments.contains("20"));
    assert!(tools[0].function.arguments.contains("add"));
281
282
283
284
}

#[tokio::test]
async fn test_json_parser_with_tokens() {
285
286
287
288
289
    let parser = JsonParser::with_config(TokenConfig {
        start_tokens: vec!["[TOOL_CALLS] [".to_string()],
        end_tokens: vec!["]".to_string()],
        separator: ", ".to_string(),
    });
290
291

    let input = r#"[TOOL_CALLS] [{"name": "search", "arguments": {"query": "rust programming"}}]"#;
292
    let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
293

294
295
    assert_eq!(tools.len(), 1);
    assert_eq!(tools[0].function.name, "search");
296
297
298
299
}

#[tokio::test]
async fn test_multiline_json_with_tokens() {
300
301
302
303
304
    let parser = JsonParser::with_config(TokenConfig {
        start_tokens: vec!["<tool>".to_string()],
        end_tokens: vec!["</tool>".to_string()],
        separator: ", ".to_string(),
    });
305
306
307
308
309
310
311
312
313
314
315

    // Pretty-printed multi-line JSON
    let input = r#"<tool>{
    "name": "get_weather",
    "arguments": {
        "location": "San Francisco",
        "units": "celsius",
        "include_forecast": true
    }
}</tool>"#;

316
317
318
319
320
321
    let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
    assert_eq!(tools.len(), 1);
    assert_eq!(tools[0].function.name, "get_weather");
    assert!(tools[0].function.arguments.contains("San Francisco"));
    assert!(tools[0].function.arguments.contains("celsius"));
    assert!(tools[0].function.arguments.contains("true"));
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
}

#[tokio::test]
async fn test_multiline_json_array() {
    let parser = JsonParser::new();

    let input = r#"[
    {
        "name": "function1",
        "arguments": {
            "param1": "value1",
            "param2": 42
        }
    },
    {
        "name": "function2",
        "parameters": {
            "data": [1, 2, 3],
            "flag": false
        }
    }
]"#;

345
346
347
348
349
350
    let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
    assert_eq!(tools.len(), 2);
    assert_eq!(tools[0].function.name, "function1");
    assert_eq!(tools[1].function.name, "function2");
    assert!(tools[0].function.arguments.contains("value1"));
    assert!(tools[1].function.arguments.contains("[1,2,3]"));
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
}

#[test]
fn test_json_parser_format_detection() {
    let parser = JsonParser::new();

    // Should detect valid tool call formats
    assert!(parser.detect_format(r#"{"name": "test", "arguments": {}}"#));
    assert!(parser.detect_format(r#"{"name": "test", "parameters": {"x": 1}}"#));
    assert!(parser.detect_format(r#"[{"name": "test"}]"#));

    // Should not detect non-tool formats
    assert!(!parser.detect_format("plain text"));
    assert!(!parser.detect_format(r#"{"key": "value"}"#));
    assert!(!parser.detect_format(r#"{"data": {"nested": true}}"#));
}

#[tokio::test]
async fn test_json_parser_streaming() {
    let parser = JsonParser::new();
    let mut state = ParseState::new();

    let full_json = r#"{"name": "get_weather", "arguments": {"location": "San Francisco"}}"#;

    let result = parser
        .parse_incremental(full_json, &mut state)
        .await
        .unwrap();

    match result {
        StreamResult::ToolComplete(tool) => {
            assert_eq!(tool.function.name, "get_weather");
            assert!(tool.function.arguments.contains("San Francisco"));
        }
        _ => panic!("Expected ToolComplete for complete JSON"),
    }
}

#[tokio::test]
async fn test_registry_with_json_parser() {
    let registry = ParserRegistry::new();

    // JSON parser should be registered by default
    assert!(registry.has_parser("json"));

    // Should get JSON parser for OpenAI models
    let parser = registry.get_parser("gpt-4-turbo").unwrap();

    let input = r#"{"name": "test", "arguments": {"x": 1}}"#;
400
401
402
    let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
    assert_eq!(tools.len(), 1);
    assert_eq!(tools[0].function.name, "test");
403
404
405
406
407
408
409
}

#[tokio::test]
async fn test_json_parser_invalid_input() {
    let parser = JsonParser::new();

    // Invalid JSON should return empty results
410
411
412
    assert_eq!(parser.parse_complete("not json").await.unwrap().1.len(), 0);
    assert_eq!(parser.parse_complete("{invalid}").await.unwrap().1.len(), 0);
    assert_eq!(parser.parse_complete("").await.unwrap().1.len(), 0);
413
414
415
416
417
418
419
420
}

#[tokio::test]
async fn test_json_parser_empty_arguments() {
    let parser = JsonParser::new();

    // Tool call with no arguments
    let input = r#"{"name": "get_time"}"#;
421
    let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
422

423
424
425
    assert_eq!(tools.len(), 1);
    assert_eq!(tools[0].function.name, "get_time");
    assert_eq!(tools[0].function.arguments, "{}");
426
427
428
429
430
431
432
433
434
435
436
437
}

#[cfg(test)]
mod failure_cases {
    use super::*;

    #[tokio::test]
    async fn test_malformed_tool_missing_name() {
        let parser = JsonParser::new();

        // Missing name field
        let input = r#"{"arguments": {"x": 1}}"#;
438
439
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 0, "Should return empty for tool without name");
440
441
442

        // Empty name
        let input = r#"{"name": "", "arguments": {"x": 1}}"#;
443
444
445
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1, "Should accept empty name string");
        assert_eq!(tools[0].function.name, "");
446
447
448
449
450
451
452
453
    }

    #[tokio::test]
    async fn test_invalid_arguments_json() {
        let parser = JsonParser::new();

        // Arguments is a string instead of object
        let input = r#"{"name": "test", "arguments": "not an object"}"#;
454
455
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
456
        // Should serialize the string as JSON
457
        assert!(tools[0].function.arguments.contains("not an object"));
458
459
460

        // Arguments is a number
        let input = r#"{"name": "test", "arguments": 42}"#;
461
462
463
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.arguments, "42");
464
465
466

        // Arguments is null
        let input = r#"{"name": "test", "arguments": null}"#;
467
468
469
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.arguments, "null");
470
471
472
473
    }

    #[tokio::test]
    async fn test_broken_wrapper_tokens() {
474
475
476
477
478
        let parser = JsonParser::with_config(TokenConfig {
            start_tokens: vec!["<tool>".to_string()],
            end_tokens: vec!["</tool>".to_string()],
            separator: ", ".to_string(),
        });
479
480
481

        // Missing end token
        let input = r#"<tool>{"name": "test", "arguments": {}}"#;
482
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
483
        assert_eq!(
484
            tools.len(),
485
486
487
488
489
490
            0,
            "Should fail to parse without complete wrapper"
        );

        // Missing start token - parser looks for complete wrapper, so this won't parse
        let input = r#"{"name": "test", "arguments": {}}</tool>"#;
491
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
492
        assert_eq!(
493
            tools.len(),
494
495
496
497
498
499
            0,
            "Should not parse JSON with incomplete wrapper"
        );

        // Mismatched tokens
        let input = r#"<tool>{"name": "test", "arguments": {}}</wrong>"#;
500
501
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 0, "Should fail with mismatched tokens");
502
503
504
505
506
507
508
509
    }

    #[tokio::test]
    async fn test_invalid_json_structures() {
        let parser = JsonParser::new();

        // Trailing comma
        let input = r#"{"name": "test", "arguments": {"x": 1,}}"#;
510
511
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 0, "Should reject JSON with trailing comma");
512
513
514

        // Missing quotes on keys
        let input = r#"{name: "test", arguments: {}}"#;
515
516
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 0, "Should reject invalid JSON syntax");
517
518
519

        // Unclosed object
        let input = r#"{"name": "test", "arguments": {"#;
520
521
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 0, "Should reject incomplete JSON");
522
523
524
525
526
527
528
529
530
531
532
533
534
    }
}

#[cfg(test)]
mod edge_cases {
    use super::*;

    #[tokio::test]
    async fn test_unicode_in_names_and_arguments() {
        let parser = JsonParser::new();

        // Unicode in function name
        let input = r#"{"name": "获取天气", "arguments": {"location": "北京"}}"#;
535
536
537
538
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.name, "获取天气");
        assert!(tools[0].function.arguments.contains("北京"));
539
540
541

        // Emoji in arguments
        let input = r#"{"name": "send_message", "arguments": {"text": "Hello 👋 World 🌍"}}"#;
542
543
544
545
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains("👋"));
        assert!(tools[0].function.arguments.contains("🌍"));
546
547
548
549
550
551
552
553
    }

    #[tokio::test]
    async fn test_escaped_characters() {
        let parser = JsonParser::new();

        // Escaped quotes in arguments
        let input = r#"{"name": "echo", "arguments": {"text": "He said \"hello\""}}"#;
554
555
556
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains(r#"\"hello\""#));
557
558
559

        // Escaped backslashes
        let input = r#"{"name": "path", "arguments": {"dir": "C:\\Users\\test"}}"#;
560
561
562
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains("\\\\"));
563
564
565

        // Newlines and tabs
        let input = r#"{"name": "format", "arguments": {"text": "line1\nline2\ttabbed"}}"#;
566
567
568
569
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains("\\n"));
        assert!(tools[0].function.arguments.contains("\\t"));
570
571
572
573
574
575
576
577
578
579
580
581
582
    }

    #[tokio::test]
    async fn test_very_large_payloads() {
        let parser = JsonParser::new();

        // Large arguments object
        let mut large_args = r#"{"name": "process", "arguments": {"#.to_string();
        for i in 0..1000 {
            large_args.push_str(&format!(r#""field_{}": "value_{}","#, i, i));
        }
        large_args.push_str(r#""final": "value"}}"#);

583
584
585
586
        let (_normal_text, tools) = parser.parse_complete(&large_args).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.name, "process");
        assert!(tools[0].function.arguments.contains("field_999"));
587
588
589
590
591
592
593
594
595
596
597

        // Large array of tool calls
        let mut large_array = "[".to_string();
        for i in 0..100 {
            if i > 0 {
                large_array.push(',');
            }
            large_array.push_str(&format!(r#"{{"name": "func_{}", "arguments": {{}}}}"#, i));
        }
        large_array.push(']');

598
599
600
        let (_normal_text, tools) = parser.parse_complete(&large_array).await.unwrap();
        assert_eq!(tools.len(), 100);
        assert_eq!(tools[99].function.name, "func_99");
601
602
603
604
605
606
607
608
609
610
611
612
613
614
    }

    #[tokio::test]
    async fn test_mixed_array_tools_and_non_tools() {
        let parser = JsonParser::new();

        // Array with both tool calls and non-tool objects
        let input = r#"[
            {"name": "tool1", "arguments": {}},
            {"not_a_tool": "just_data"},
            {"name": "tool2", "parameters": {"x": 1}},
            {"key": "value", "another": "field"}
        ]"#;

615
616
617
618
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 2, "Should only parse valid tool calls");
        assert_eq!(tools[0].function.name, "tool1");
        assert_eq!(tools[1].function.name, "tool2");
619
620
621
622
623
624
625
626
    }

    #[tokio::test]
    async fn test_duplicate_keys_in_json() {
        let parser = JsonParser::new();

        // JSON with duplicate keys (last one wins in most parsers)
        let input = r#"{"name": "first", "name": "second", "arguments": {"x": 1, "x": 2}}"#;
627
628
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
629
        assert_eq!(
630
            tools[0].function.name, "second",
631
632
633
            "Last duplicate key should win"
        );
        assert!(
634
            tools[0].function.arguments.contains("2"),
635
636
637
638
639
640
641
642
643
644
            "Last duplicate value should win"
        );
    }

    #[tokio::test]
    async fn test_null_values_in_arguments() {
        let parser = JsonParser::new();

        // Null values in arguments
        let input = r#"{"name": "test", "arguments": {"required": "value", "optional": null}}"#;
645
646
647
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains("null"));
648
649
650

        // Array with null
        let input = r#"{"name": "test", "arguments": {"items": [1, null, "three"]}}"#;
651
652
653
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains("null"));
654
655
656
657
    }

    #[tokio::test]
    async fn test_multiple_token_pairs_with_conflicts() {
658
659
660
661
662
        let parser = JsonParser::with_config(TokenConfig {
            start_tokens: vec!["<<".to_string(), "<tool>".to_string()],
            end_tokens: vec![">>".to_string(), "</tool>".to_string()],
            separator: ", ".to_string(),
        });
663
664
665

        // First pattern
        let input = r#"<<{"name": "test1", "arguments": {}}>>"#;
666
667
668
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.name, "test1");
669
670
671

        // Second pattern
        let input = r#"<tool>{"name": "test2", "arguments": {}}</tool>"#;
672
673
674
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.name, "test2");
675
676
677

        // Nested patterns (should use first match)
        let input = r#"<<tool>{"name": "test3", "arguments": {}}</tool>>"#;
678
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
679
680
        // This is tricky - depends on regex behavior
        // The parser should handle this gracefully
681
        assert!(tools.len() <= 1, "Should not parse multiple times");
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
    }

    #[tokio::test]
    async fn test_streaming_with_partial_chunks() {
        let parser = JsonParser::new();

        let mut state1 = ParseState::new();
        let partial = r#"{"#;
        let result = parser
            .parse_incremental(partial, &mut state1)
            .await
            .unwrap();
        assert!(
            matches!(result, StreamResult::Incomplete),
            "Should return Incomplete for just opening brace"
        );

        let mut state2 = ParseState::new();
        let complete = r#"{"name": "get_weather", "arguments": {"location": "SF"}}"#;
        let result = parser
            .parse_incremental(complete, &mut state2)
            .await
            .unwrap();

        match result {
            StreamResult::ToolComplete(tool) => {
                assert_eq!(tool.function.name, "get_weather");
                let args: serde_json::Value =
                    serde_json::from_str(&tool.function.arguments).unwrap();
                assert_eq!(args["location"], "SF");
            }
            _ => panic!("Expected ToolComplete for complete JSON"),
        }

        // The PartialJson parser can complete partial JSON by filling in missing values
        let mut state3 = ParseState::new();
        let partial_with_name = r#"{"name": "test", "argum"#;
        let result = parser
            .parse_incremental(partial_with_name, &mut state3)
            .await
            .unwrap();

        match result {
            StreamResult::ToolComplete(tool) => {
                assert_eq!(tool.function.name, "test");
                // Arguments will be empty object since "argum" is incomplete
                assert_eq!(tool.function.arguments, "{}");
            }
            StreamResult::ToolName { name, .. } => {
                assert_eq!(name, "test");
            }
            StreamResult::Incomplete => {
                // Also acceptable if parser decides to wait
            }
            _ => panic!("Unexpected result for partial JSON with name"),
        }
    }

    #[tokio::test]
    async fn test_special_json_values() {
        let parser = JsonParser::new();

        // Boolean values
        let input = r#"{"name": "toggle", "arguments": {"enabled": true, "disabled": false}}"#;
746
747
748
749
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains("true"));
        assert!(tools[0].function.arguments.contains("false"));
750
751
752

        // Numbers (including float and negative)
        let input = r#"{"name": "calc", "arguments": {"int": 42, "float": 3.14, "negative": -17}}"#;
753
754
755
756
757
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains("42"));
        assert!(tools[0].function.arguments.contains("3.14"));
        assert!(tools[0].function.arguments.contains("-17"));
758
759
760

        // Empty arrays and objects
        let input = r#"{"name": "test", "arguments": {"empty_arr": [], "empty_obj": {}}}"#;
761
762
763
764
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains("[]"));
        assert!(tools[0].function.arguments.contains("{}"));
765
766
767
768
769
770
771
772
    }

    #[tokio::test]
    async fn test_function_field_alternative() {
        let parser = JsonParser::new();

        // Using "function" instead of "name"
        let input = r#"{"function": "test_func", "arguments": {"x": 1}}"#;
773
774
775
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.name, "test_func");
776
777
778

        // Both "name" and "function" present (name should take precedence)
        let input = r#"{"name": "primary", "function": "secondary", "arguments": {}}"#;
779
780
781
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.name, "primary");
782
783
784
785
786
787
788
789
790
791
792
793
794
    }

    #[tokio::test]
    async fn test_whitespace_handling() {
        let parser = JsonParser::new();

        // Extra whitespace everywhere
        let input = r#"  {
            "name"   :   "test"  ,
            "arguments"   :   {
                "key"   :   "value"
            }
        }  "#;
795
796
797
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.name, "test");
798
799
800

        // Minified JSON (no whitespace)
        let input = r#"{"name":"compact","arguments":{"a":1,"b":2}}"#;
801
802
803
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert_eq!(tools[0].function.name, "compact");
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
    }
}

#[cfg(test)]
mod stress_tests {
    use super::*;

    #[tokio::test]
    async fn test_deeply_nested_arguments() {
        let parser = JsonParser::new();

        // Deeply nested structure
        let input = r#"{
            "name": "nested",
            "arguments": {
                "level1": {
                    "level2": {
                        "level3": {
                            "level4": {
                                "level5": {
                                    "value": "deep"
                                }
                            }
                        }
                    }
                }
            }
        }"#;

833
834
835
        let (_normal_text, tools) = parser.parse_complete(input).await.unwrap();
        assert_eq!(tools.len(), 1);
        assert!(tools[0].function.arguments.contains("deep"));
836
837
838
839
840
841
842
843
844
845
846
847
    }

    #[tokio::test]
    async fn test_concurrent_parser_usage() {
        let parser = std::sync::Arc::new(JsonParser::new());

        let mut handles = vec![];

        for i in 0..10 {
            let parser_clone = parser.clone();
            let handle = tokio::spawn(async move {
                let input = format!(r#"{{"name": "func_{}", "arguments": {{}}}}"#, i);
848
849
850
                let (_normal_text, tools) = parser_clone.parse_complete(&input).await.unwrap();
                assert_eq!(tools.len(), 1);
                assert_eq!(tools[0].function.name, format!("func_{}", i));
851
852
853
854
855
856
857
858
859
            });
            handles.push(handle);
        }

        for handle in handles {
            handle.await.unwrap();
        }
    }
}