"vscode:/vscode.git/clone" did not exist on "32b4c63f02b2ab28a49a040b1d170a903a5cd9dc"
test_mistral_reasoning_parser.py 9.82 KB
Newer Older
Julien Denize's avatar
Julien Denize committed
1
2
3
4
5
6
7
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import pytest

from tests.reasoning.utils import run_reasoning_extraction_mistral
from vllm.reasoning import ReasoningParser, ReasoningParserManager
8
from vllm.tokenizers.mistral import MistralTokenizer
Julien Denize's avatar
Julien Denize committed
9
10
11
12
13
14
15

parser_name = "mistral"


@pytest.fixture(scope="module")
def mistral_tokenizer():
    mistral_tokenizer = MistralTokenizer.from_pretrained(
16
        "mistralai/Magistral-Small-2509"
17
    )
Julien Denize's avatar
Julien Denize committed
18
19
20
    return mistral_tokenizer


21
INVALID_SIMPLE_REASONING = {
Julien Denize's avatar
Julien Denize committed
22
    "output": "This is a reasoning section[/THINK]This is the rest",
23
24
25
    "reasoning": None,
    "content": "This is a reasoning sectionThis is the rest",
    "is_reasoning_end": False,
Julien Denize's avatar
Julien Denize committed
26
}
27
INVALID_COMPLETE_REASONING = {
Julien Denize's avatar
Julien Denize committed
28
    "output": "This is a reasoning section[/THINK]",
29
30
31
    "reasoning": None,
    "content": "This is a reasoning section",
    "is_reasoning_end": False,
Julien Denize's avatar
Julien Denize committed
32
33
}
NO_CONTENT = {
34
35
    "output": "[THINK]This is reasoning",
    "reasoning": "This is reasoning",
Julien Denize's avatar
Julien Denize committed
36
37
38
    "content": None,
    "is_reasoning_end": False,
}
39
40
41
42
43
44
NO_REASONING = {
    "output": "This is content",
    "reasoning": None,
    "content": "This is content",
    "is_reasoning_end": False,
}
Julien Denize's avatar
Julien Denize committed
45
46
NO_REASONING_STREAMING = {
    "output": "This is a reasoning section",
47
48
    "reasoning": None,
    "content": "This is a reasoning section",
Julien Denize's avatar
Julien Denize committed
49
50
    "is_reasoning_end": False,
}
51
INVALID_MULTIPLE_LINES = {
Julien Denize's avatar
Julien Denize committed
52
    "output": "This\nThat[/THINK]This is the rest\nThat",
53
54
55
    "reasoning": None,
    "content": "This\nThatThis is the rest\nThat",
    "is_reasoning_end": False,
Julien Denize's avatar
Julien Denize committed
56
}
57
INVALID_SHORTEST_REASONING_NO_STREAMING = {
Julien Denize's avatar
Julien Denize committed
58
    "output": "[/THINK]This is the rest",
59
    "reasoning": None,
Julien Denize's avatar
Julien Denize committed
60
    "content": "This is the rest",
61
    "is_reasoning_end": False,
Julien Denize's avatar
Julien Denize committed
62
}
63
INVALID_SHORTEST_REASONING = {
Julien Denize's avatar
Julien Denize committed
64
    "output": "[/THINK]This is the rest",
65
    "reasoning": None,
Julien Denize's avatar
Julien Denize committed
66
    "content": "This is the rest",
67
    "is_reasoning_end": False,
Julien Denize's avatar
Julien Denize committed
68
69
70
}
REASONING_WITH_THINK = {
    "output": "[THINK]This is a reasoning section[/THINK]This is the rest",
71
    "reasoning": "This is a reasoning section",
Julien Denize's avatar
Julien Denize committed
72
73
74
75
76
    "content": "This is the rest",
    "is_reasoning_end": True,
}
COMPLETE_REASONING_WITH_THINK = {
    "output": "[THINK]This is a reasoning section[/THINK]",
77
    "reasoning": "This is a reasoning section",
Julien Denize's avatar
Julien Denize committed
78
79
80
81
82
    "content": None,
    "is_reasoning_end": True,
}
MULTIPLE_LINES_WITH_THINK = {
    "output": "[THINK]This\nThat[/THINK]This is the rest\nThat",
83
    "reasoning": "This\nThat",
Julien Denize's avatar
Julien Denize committed
84
85
86
    "content": "This is the rest\nThat",
    "is_reasoning_end": True,
}
87
INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK = {
Julien Denize's avatar
Julien Denize committed
88
    "output": "[/THINK]This is the rest",
89
    "reasoning": None,
Julien Denize's avatar
Julien Denize committed
90
    "content": "This is the rest",
91
    "is_reasoning_end": False,
Julien Denize's avatar
Julien Denize committed
92
}
93
INVALID_SHORTEST_REASONING_WITH_THINK = {
Julien Denize's avatar
Julien Denize committed
94
    "output": "[/THINK]This is the rest",
95
    "reasoning": None,
Julien Denize's avatar
Julien Denize committed
96
    "content": "This is the rest",
97
    "is_reasoning_end": False,
Julien Denize's avatar
Julien Denize committed
98
99
100
}
THINK_NO_END = {
    "output": "[THINK]This is a reasoning section",
101
    "reasoning": "This is a reasoning section",
Julien Denize's avatar
Julien Denize committed
102
103
104
105
106
    "content": None,
    "is_reasoning_end": False,
}
EMPTY = {
    "output": "",
107
108
    "reasoning": None,
    "content": "",
Julien Denize's avatar
Julien Denize committed
109
110
111
112
    "is_reasoning_end": False,
}
EMPTY_STREAMING = {
    "output": "",
113
    "reasoning": None,
Julien Denize's avatar
Julien Denize committed
114
115
116
117
    "content": None,
    "is_reasoning_end": False,
}
NEW_LINE = {
118
    "output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
119
    "reasoning": "This is a reasoning section",
120
    "content": "Before\n\nThis is the rest",
Julien Denize's avatar
Julien Denize committed
121
122
123
    "is_reasoning_end": True,
}
NEW_LINE_STREAMING = {
124
125
126
    "output": "Before\n[THINK]This is a reasoning section[/THINK]\nThis is the rest",
    "reasoning": "This is a reasoning section",
    "content": "Before\n\nThis is the rest",
Julien Denize's avatar
Julien Denize committed
127
128
129
130
131
132
    "is_reasoning_end": True,
}

TEST_CASES = [
    pytest.param(
        False,
133
134
        INVALID_SIMPLE_REASONING,
        id="invalid_simple_reasoning",
Julien Denize's avatar
Julien Denize committed
135
136
137
    ),
    pytest.param(
        True,
138
139
        INVALID_SIMPLE_REASONING,
        id="invalid_simple_reasoning_streaming",
Julien Denize's avatar
Julien Denize committed
140
141
142
    ),
    pytest.param(
        False,
143
144
        INVALID_COMPLETE_REASONING,
        id="invalid_complete_reasoning",
Julien Denize's avatar
Julien Denize committed
145
146
147
    ),
    pytest.param(
        True,
148
149
        INVALID_COMPLETE_REASONING,
        id="invalid_complete_reasoning_streaming",
Julien Denize's avatar
Julien Denize committed
150
151
152
153
    ),
    pytest.param(
        False,
        NO_CONTENT,
154
155
156
157
158
159
        id="no_content",
    ),
    pytest.param(
        False,
        NO_REASONING,
        id="no_reasoning",
Julien Denize's avatar
Julien Denize committed
160
161
162
163
164
165
166
167
    ),
    pytest.param(
        True,
        NO_REASONING_STREAMING,
        id="no_reasoning_token_streaming",
    ),
    pytest.param(
        False,
168
169
        INVALID_MULTIPLE_LINES,
        id="invalid_multiple_lines",
Julien Denize's avatar
Julien Denize committed
170
171
172
    ),
    pytest.param(
        True,
173
174
        INVALID_MULTIPLE_LINES,
        id="invalid_multiple_lines_streaming",
Julien Denize's avatar
Julien Denize committed
175
176
177
    ),
    pytest.param(
        True,
178
179
        INVALID_SHORTEST_REASONING,
        id="invalid_shortest",
Julien Denize's avatar
Julien Denize committed
180
181
182
    ),
    pytest.param(
        False,
183
184
        INVALID_SHORTEST_REASONING_NO_STREAMING,
        id="invalid_shortest_streaming",
Julien Denize's avatar
Julien Denize committed
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
    ),
    pytest.param(
        False,
        REASONING_WITH_THINK,
        id="reasoning_with_think",
    ),
    pytest.param(
        True,
        REASONING_WITH_THINK,
        id="reasoning_with_think_streaming",
    ),
    pytest.param(
        False,
        COMPLETE_REASONING_WITH_THINK,
        id="complete_reasoning_with_think",
    ),
    pytest.param(
        True,
        COMPLETE_REASONING_WITH_THINK,
        id="complete_reasoning_with_think_streaming",
    ),
    pytest.param(
        False,
        MULTIPLE_LINES_WITH_THINK,
        id="multiple_lines_with_think",
    ),
    pytest.param(
        True,
        MULTIPLE_LINES_WITH_THINK,
        id="multiple_lines_with_think_streaming",
    ),
    pytest.param(
        False,
218
219
        INVALID_SHORTEST_REASONING_NO_STREAMING_WITH_THINK,
        id="invalid_shortest_with_think",
Julien Denize's avatar
Julien Denize committed
220
221
222
    ),
    pytest.param(
        True,
223
224
        INVALID_SHORTEST_REASONING_WITH_THINK,
        id="invalid_shortest_with_think_streaming",
Julien Denize's avatar
Julien Denize committed
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
    ),
    pytest.param(
        False,
        THINK_NO_END,
        id="think_no_end",
    ),
    pytest.param(
        True,
        THINK_NO_END,
        id="think_no_end_streaming",
    ),
    pytest.param(
        False,
        EMPTY,
        id="empty",
    ),
    pytest.param(
        True,
        EMPTY_STREAMING,
        id="empty_streaming",
    ),
    pytest.param(
        False,
        NEW_LINE,
        id="new_line",
    ),
    pytest.param(
        True,
        NEW_LINE_STREAMING,
        id="new_line_streaming",
    ),
]


@pytest.mark.parametrize("streaming, param_dict", TEST_CASES)
def test_mistral_reasoning(
    streaming: bool,
    param_dict: dict,
    mistral_tokenizer: MistralTokenizer,
):
    output = param_dict["output"]

    index_think = output.find("[THINK]")
    len_think = len("[THINK]")
    index_end_think = output.find("[/THINK]")
    len_end_think = len("[/THINK]")

    # encode everything to tokens ids
    output_tokens = []
    if index_think != -1:
        output_before_think = output[:index_think]
        output_tokens += mistral_tokenizer.tokenizer.encode(
277
278
            output_before_think, False, False
        )
Julien Denize's avatar
Julien Denize committed
279
280
281
        output_tokens += [mistral_tokenizer.instruct.BEGIN_THINK]

        if index_end_think != -1:
282
283
            output_middle = output[index_think + len_think : index_end_think]
            output_after_think = output[index_end_think + len_end_think :]
Julien Denize's avatar
Julien Denize committed
284
            output_tokens += mistral_tokenizer.tokenizer.encode(
285
286
                output_middle, False, False
            )
Julien Denize's avatar
Julien Denize committed
287
288
            output_tokens += [mistral_tokenizer.instruct.END_THINK]
            output_tokens += mistral_tokenizer.tokenizer.encode(
289
290
                output_after_think, False, False
            )
Julien Denize's avatar
Julien Denize committed
291
        else:
292
            output_middle = output[index_think + len_think :]
Julien Denize's avatar
Julien Denize committed
293
            output_tokens += mistral_tokenizer.tokenizer.encode(
294
295
                output_middle, False, False
            )
Julien Denize's avatar
Julien Denize committed
296
297
    elif index_end_think != -1:
        output_before_think = output[:index_end_think]
298
        output_after_think = output[index_end_think + len_end_think :]
Julien Denize's avatar
Julien Denize committed
299
        output_tokens += mistral_tokenizer.tokenizer.encode(
300
301
            output_before_think, False, False
        )
Julien Denize's avatar
Julien Denize committed
302
303
        output_tokens += [mistral_tokenizer.instruct.END_THINK]
        output_tokens += mistral_tokenizer.tokenizer.encode(
304
305
            output_after_think, False, False
        )
Julien Denize's avatar
Julien Denize committed
306
    else:
307
        output_tokens += mistral_tokenizer.tokenizer.encode(output, False, False)
Julien Denize's avatar
Julien Denize committed
308

309
310
311
    parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)(
        mistral_tokenizer
    )
Julien Denize's avatar
Julien Denize committed
312

313
314
315
    reasoning, content = run_reasoning_extraction_mistral(
        parser, output_tokens, streaming=streaming
    )
Julien Denize's avatar
Julien Denize committed
316

317
    assert reasoning == param_dict["reasoning"]
Julien Denize's avatar
Julien Denize committed
318
319
320
321
322
323
324
325
    assert content == param_dict["content"]

    # Test is_reasoning_end
    is_reasoning_end = parser.is_reasoning_end(output_tokens)
    assert is_reasoning_end == param_dict["is_reasoning_end"]

    # Test extract_content
    if param_dict["content"] is not None:
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
        # Handle the case where there are tokens outputted before Thinking.
        # This should not occur if the model is well trained and prompted.
        if "[THINK]" in param_dict["output"] and not param_dict["output"].startswith(
            "[THINK]"
        ):
            before_content = param_dict["output"].split("[THINK]")[0]
            before_token_ids = mistral_tokenizer.tokenizer.encode(
                before_content, bos=False, eos=False
            )
            left_to_encode = param_dict["content"][len(before_content) :]
        # Normal situation.
        else:
            before_token_ids = []
            left_to_encode = param_dict["content"]

        content_tokens = parser.extract_content_ids(output_tokens)
        expected_token_ids = before_token_ids + mistral_tokenizer.tokenizer.encode(
            left_to_encode, bos=False, eos=False
344
        )
345
        assert content_tokens == expected_token_ids
Julien Denize's avatar
Julien Denize committed
346
347
348
    else:
        content = parser.extract_content_ids(output_tokens)
        assert content == []