Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
test_step3_tool_parser.py 5.27 KB
Newer Older
Ben Browning's avatar
[CI/Build] Add common tool call parser test suite (#27599)  
Ben Browning committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112 
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project


import pytest

from tests.tool_parsers.common_tests import (
    ToolParserTestConfig,
    ToolParserTests,
)
from vllm.tokenizers import TokenizerLike, get_tokenizer


class TestStep3ToolParser(ToolParserTests):
    @pytest.fixture(scope="class")
    def tokenizer(self) -> TokenizerLike:
        return get_tokenizer("stepfun-ai/step3")

    @pytest.fixture
    def test_config(self) -> ToolParserTestConfig:
        return ToolParserTestConfig(
            parser_name="step3",
            # Test data
            no_tool_calls_output="This is a regular response without any tool calls.",
            single_tool_call_output=(
                "<｜tool_calls_begin｜><｜tool_call_begin｜>"
                '<steptml:invoke name="get_weather">'
                '<steptml:parameter name="city">Tokyo</steptml:parameter>'
                "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>"
            ),
            parallel_tool_calls_output=(
                "<｜tool_calls_begin｜><｜tool_call_begin｜>"
                '<steptml:invoke name="get_weather">'
                '<steptml:parameter name="city">Tokyo</steptml:parameter>'
                "</steptml:invoke><｜tool_call_end｜><｜tool_sep｜>"
                '<｜tool_call_begin｜><steptml:invoke name="get_time">'
                '<steptml:parameter name="timezone">Asia/Tokyo</steptml:parameter>'
                "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>"
            ),
            various_data_types_output=(
                "<｜tool_calls_begin｜><｜tool_call_begin｜>"
                '<steptml:invoke name="test_function">'
                '<steptml:parameter name="string_field">hello</steptml:parameter>'
                '<steptml:parameter name="int_field">42</steptml:parameter>'
                '<steptml:parameter name="float_field">3.14</steptml:parameter>'
                '<steptml:parameter name="bool_field">true</steptml:parameter>'
                '<steptml:parameter name="null_field">null</steptml:parameter>'
                '<steptml:parameter name="array_field">'
                '["a", "b", "c"]</steptml:parameter>'
                '<steptml:parameter name="object_field">'
                '{"nested": "value"}</steptml:parameter>'
                "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>"
            ),
            empty_arguments_output=(
                "<｜tool_calls_begin｜><｜tool_call_begin｜>"
                '<steptml:invoke name="refresh"></steptml:invoke>'
                "<｜tool_call_end｜><｜tool_calls_end｜>"
            ),
            surrounding_text_output=(
                "Let me check the weather for you.\n\n"
                "<｜tool_calls_begin｜><｜tool_call_begin｜>"
                '<steptml:invoke name="get_weather">'
                '<steptml:parameter name="city">Tokyo</steptml:parameter>'
                "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>\n\n"
                "I'll get that information."
            ),
            escaped_strings_output=(
                "<｜tool_calls_begin｜><｜tool_call_begin｜>"
                '<steptml:invoke name="test_function">'
                '<steptml:parameter name="quoted">He said "hello"</steptml:parameter>'
                '<steptml:parameter name="path">C:\\Users\\file.txt</steptml:parameter>'
                '<steptml:parameter name="newline">line1\nline2</steptml:parameter>'
                "</steptml:invoke><｜tool_call_end｜><｜tool_calls_end｜>"
            ),
            malformed_input_outputs=[
                (
                    "<｜tool_calls_begin｜><｜tool_call_begin｜>"
                    '<steptml:invoke name="func">'
                ),
                (
                    '<｜tool_call_begin｜><steptml:invoke name="func">'
                    "</steptml:invoke><｜tool_call_end｜>"
                ),
            ],
            # Expected results
            single_tool_call_expected_name="get_weather",
            single_tool_call_expected_args={"city": "Tokyo"},
            parallel_tool_calls_count=2,
            parallel_tool_calls_names=["get_weather", "get_time"],
            # xfail markers
            xfail_nonstreaming={
                "test_single_tool_call_simple_args": (
                    "Step3 parser non-streaming has bugs"
                ),
                "test_parallel_tool_calls": ("Step3 parser non-streaming has bugs"),
                "test_various_data_types": "Step3 parser non-streaming has bugs",
                "test_empty_arguments": "Step3 parser non-streaming has bugs",
                "test_surrounding_text": "Step3 parser non-streaming has bugs",
                "test_escaped_strings": "Step3 parser non-streaming has bugs",
            },
            xfail_streaming={
                "test_parallel_tool_calls": (
                    "Step3 parser has significant bugs in both streaming "
                    "and non-streaming"
                ),
                "test_streaming_reconstruction": (
                    "Step3 parser non-streaming has bugs, so streaming "
                    "doesn't match non-streaming"
                ),
            },
            supports_typed_arguments=False,
        )