test_chat.py 28.7 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
# imports for structured outputs tests
5
6
7
8
9
import json

import jsonschema
import openai  # use the official client for correctness check
import pytest
10
import pytest_asyncio
11
import regex as re
12
import requests
13
import torch
14
from openai import BadRequestError
15

16
from ...utils import RemoteOpenAIServer
17
18
19
20
21

# any model with a chat template should work here
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"


22
@pytest.fixture(scope="module")
23
def server(zephyr_lora_files):  # noqa: F811
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
    args = [
        # use half precision for speed and memory savings in CI environment
        "--dtype",
        "bfloat16",
        "--max-model-len",
        "8192",
        "--enforce-eager",
        # lora config below
        "--enable-lora",
        "--lora-modules",
        f"zephyr-lora={zephyr_lora_files}",
        "--max-lora-rank",
        "64",
        "--max-cpu-loras",
        "2",
        "--max-num-seqs",
        "128",
    ]

    with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
44
        yield remote_server
45
46


47
48
49
50
@pytest_asyncio.fixture
async def client(server):
    async with server.get_async_client() as async_client:
        yield async_client
51
52
53
54
55
56


@pytest.mark.asyncio
@pytest.mark.parametrize(
    # first test base model, then test loras
    "model_name",
57
    [MODEL_NAME, "zephyr-lora"],
58
59
)
async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
60
61
62
63
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
64

65
66
67
68
69
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=5,
        temperature=0.0,
70
71
        logprobs=False,
    )
72
73
74
75
76
77
78
79
80
81
82
83

    choice = chat_completion.choices[0]
    assert choice.logprobs is None


@pytest.mark.asyncio
@pytest.mark.parametrize(
    # just test 1 lora hereafter
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
84
85
86
87
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
88

89
90
91
92
93
94
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=5,
        temperature=0.0,
        logprobs=True,
95
96
        top_logprobs=0,
    )
97
98
99
100
101
102
103
104
105
106
107
108
109

    choice = chat_completion.choices[0]
    assert choice.logprobs is not None
    assert choice.logprobs.content is not None
    assert len(choice.logprobs.content[0].top_logprobs) == 0


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
110
111
112
113
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
114

115
116
117
118
119
120
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=5,
        temperature=0.0,
        logprobs=True,
121
122
        top_logprobs=5,
    )
123
124
125
126
127
128
129
130
131
132
133
134

    choice = chat_completion.choices[0]
    assert choice.logprobs is not None
    assert choice.logprobs.content is not None
    assert len(choice.logprobs.content[0].top_logprobs) == 5


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
135
136
137
138
139
async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI, model_name: str):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
140
141
142

    # Default max_logprobs is 20, so this should raise an error
    with pytest.raises((openai.BadRequestError, openai.APIError)):
143
144
145
146
147
148
149
150
        stream = await client.chat.completions.create(
            model=model_name,
            messages=messages,
            max_completion_tokens=10,
            logprobs=True,
            top_logprobs=21,
            stream=True,
        )
151
152
153
154
        async for chunk in stream:
            ...

    with pytest.raises(openai.BadRequestError):
155
156
157
158
159
160
161
162
        await client.chat.completions.create(
            model=model_name,
            messages=messages,
            max_completion_tokens=10,
            logprobs=True,
            top_logprobs=30,
            stream=False,
        )
163
164

    # the server should still work afterwards
165
    chat_completion = await client.chat.completions.create(
166
167
        model=model_name, messages=messages, max_completion_tokens=10, stream=False
    )
168
169
170
171
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 0


172
173
174
175
176
@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name, prompt_logprobs",
    [(MODEL_NAME, 1), (MODEL_NAME, 0), (MODEL_NAME, -1), (MODEL_NAME, None)],
)
177
async def test_prompt_logprobs_chat(
178
    client: openai.AsyncOpenAI, model_name: str, prompt_logprobs: int | None
179
):
180
    params: dict = {
181
182
183
184
185
186
187
188
189
190
        "messages": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Who won the world series in 2020?"},
            {
                "role": "assistant",
                "content": "The Los Angeles Dodgers won the World Series in 2020.",
            },
            {"role": "user", "content": "Where was it played?"},
        ],
        "model": model_name,
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
    }

    if prompt_logprobs is not None:
        params["extra_body"] = {"prompt_logprobs": prompt_logprobs}

    if prompt_logprobs is not None and prompt_logprobs < 0:
        with pytest.raises(BadRequestError):
            await client.chat.completions.create(**params)
    else:
        completion = await client.chat.completions.create(**params)
        if prompt_logprobs is not None:
            assert completion.prompt_logprobs is not None
            assert len(completion.prompt_logprobs) > 0
        else:
            assert completion.prompt_logprobs is None


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME],
)
213
214
215
async def test_more_than_one_prompt_logprobs_chat(
    client: openai.AsyncOpenAI, model_name: str
):
216
    params: dict = {
217
218
219
220
221
222
223
224
225
226
227
        "messages": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Who won the world series in 2020?"},
            {
                "role": "assistant",
                "content": "The Los Angeles Dodgers won the World Series in 2020.",
            },
            {"role": "user", "content": "Where was it played?"},
        ],
        "model": model_name,
        "extra_body": {"prompt_logprobs": 1},
228
229
230
231
232
233
234
235
236
237
238
    }

    completion_1 = await client.chat.completions.create(**params)

    params["extra_body"] = {"prompt_logprobs": 2}
    completion_2 = await client.chat.completions.create(**params)

    assert len(completion_1.prompt_logprobs[3]) == 1
    assert len(completion_2.prompt_logprobs[3]) == 2


239
240
241
242
243
@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
244
245
246
247
248
async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
249
250

    # test single completion
251
252
253
254
255
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
        logprobs=True,
256
257
        top_logprobs=5,
    )
258
259
260
261
262
263
    assert chat_completion.id is not None
    assert len(chat_completion.choices) == 1

    choice = chat_completion.choices[0]
    assert choice.finish_reason == "length"
    assert chat_completion.usage == openai.types.CompletionUsage(
264
265
        completion_tokens=10, prompt_tokens=37, total_tokens=47
    )
266
267
268
269
270
271
272
273
274
275
276

    message = choice.message
    assert message.content is not None and len(message.content) >= 10
    assert message.role == "assistant"
    messages.append({"role": "assistant", "content": message.content})

    # test multi-turn dialogue
    messages.append({"role": "user", "content": "express your result in json"})
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
277
        max_completion_tokens=10,
278
279
280
281
282
283
284
285
286
287
288
289
    )
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 0


@pytest.mark.asyncio
@pytest.mark.parametrize(
    # just test 1 lora hereafter
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
290
291
292
293
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
294
295
296
297
298

    # test single completion
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
299
        max_completion_tokens=10,
300
301
302
303
304
305
306
307
308
        temperature=0.0,
    )
    output = chat_completion.choices[0].message.content
    stop_reason = chat_completion.choices[0].finish_reason

    # test streaming
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
309
        max_completion_tokens=10,
310
311
312
        temperature=0.0,
        stream=True,
    )
313
    chunks: list[str] = []
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
    finish_reason_count = 0
    async for chunk in stream:
        delta = chunk.choices[0].delta
        if delta.role:
            assert delta.role == "assistant"
        if delta.content:
            chunks.append(delta.content)
        if chunk.choices[0].finish_reason is not None:
            finish_reason_count += 1
    # finish reason should only return in last block
    assert finish_reason_count == 1
    assert chunk.choices[0].finish_reason == stop_reason
    assert delta.content
    assert "".join(chunks) == output


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
)
335
336
337
338
339
340
341
async def test_chat_completion_stream_options(
    client: openai.AsyncOpenAI, model_name: str
):
    messages = [
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "What is the capital of France?"},
    ]
342
343
344
345
346

    # Test stream=True, stream_options={"include_usage": False}
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
347
        max_completion_tokens=10,
348
349
        temperature=0.0,
        stream=True,
350
351
        stream_options={"include_usage": False},
    )
352
353
354
    async for chunk in stream:
        assert chunk.usage is None

355
356
    # Test stream=True, stream_options={"include_usage": True,
    #                                   "continuous_usage_stats": False}}
357
358
359
360
361
362
363
364
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
        temperature=0.0,
        stream=True,
        stream_options={"include_usage": True, "continuous_usage_stats": False},
    )
365
366
367
368
369
370

    async for chunk in stream:
        if chunk.choices[0].finish_reason is None:
            assert chunk.usage is None
        else:
            assert chunk.usage is None
371
            final_chunk = await anext(stream)
372
373
374
375
            assert final_chunk.usage is not None
            assert final_chunk.usage.prompt_tokens > 0
            assert final_chunk.usage.completion_tokens > 0
            assert final_chunk.usage.total_tokens == (
376
377
                final_chunk.usage.prompt_tokens + final_chunk.usage.completion_tokens
            )
378
379
380
381
382
383
384
            assert final_chunk.choices == []

    # Test stream=False, stream_options={"include_usage": None}
    with pytest.raises(BadRequestError):
        await client.chat.completions.create(
            model=model_name,
            messages=messages,
385
            max_completion_tokens=10,
386
387
            temperature=0.0,
            stream=False,
388
389
            stream_options={"include_usage": None},
        )
390
391
392
393
394
395

    # Test stream=False, stream_options={"include_usage": True}
    with pytest.raises(BadRequestError):
        await client.chat.completions.create(
            model=model_name,
            messages=messages,
396
            max_completion_tokens=10,
397
398
            temperature=0.0,
            stream=False,
399
400
            stream_options={"include_usage": True},
        )
401

402
403
404
405
406
    # Test stream=True, stream_options={"include_usage": True,
    #                           "continuous_usage_stats": True}
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
407
        max_completion_tokens=10,
408
        extra_body=dict(min_tokens=10),
409
410
411
412
        temperature=0.0,
        stream=True,
        stream_options={
            "include_usage": True,
413
            "continuous_usage_stats": True,
414
415
        },
    )
416
    last_completion_tokens = 0
417
418
    async for chunk in stream:
        assert chunk.usage.prompt_tokens >= 0
419
420
421
422
423
424
425
426
427
428
429
        assert (
            last_completion_tokens == 0
            or chunk.usage.completion_tokens > last_completion_tokens
            or (
                not chunk.choices
                and chunk.usage.completion_tokens == last_completion_tokens
            )
        )
        assert chunk.usage.total_tokens == (
            chunk.usage.prompt_tokens + chunk.usage.completion_tokens
        )
430
431
432
        last_completion_tokens = chunk.usage.completion_tokens

    assert last_completion_tokens == 10
433

434
435

@pytest.mark.asyncio
436
async def test_structured_outputs_choice_chat(
437
438
439
    client: openai.AsyncOpenAI,
    sample_structured_outputs_choices,
):
440
441
442
443
444
445
446
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": "The best language for type-safe systems programming is ",
        },
    ]
447
448
449
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
450
        max_completion_tokens=10,
451
        temperature=0.7,
452
        extra_body=dict(
453
454
455
            structured_outputs={"choice": sample_structured_outputs_choices}
        ),
    )
456
    choice1 = chat_completion.choices[0].message.content
457
    assert choice1 in sample_structured_outputs_choices
458
459

    messages.append({"role": "assistant", "content": choice1})
460
    messages.append({"role": "user", "content": "I disagree, pick another one"})
461
462
463
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
464
        max_completion_tokens=10,
465
        temperature=0.7,
466
        extra_body=dict(
467
468
469
            structured_outputs={"choice": sample_structured_outputs_choices}
        ),
    )
470
    choice2 = chat_completion.choices[0].message.content
471
    assert choice2 in sample_structured_outputs_choices
472
473
474
475
    assert choice1 != choice2


@pytest.mark.asyncio
476
477
478
479
async def test_structured_outputs_json_chat(
    client: openai.AsyncOpenAI,
    sample_json_schema,
):
480
481
482
483
484
485
486
487
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": f"Give an example JSON for an employee profile that "
            f"fits this schema: {sample_json_schema}",
        },
    ]
488
489
490
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
491
        max_completion_tokens=1000,
492
493
        extra_body=dict(structured_outputs={"json": sample_json_schema}),
    )
494
495
496
    message = chat_completion.choices[0].message
    assert message.content is not None
    json1 = json.loads(message.content)
497
    jsonschema.validate(instance=json1, schema=sample_json_schema)
498
499

    messages.append({"role": "assistant", "content": message.content})
500
501
502
    messages.append(
        {"role": "user", "content": "Give me another one with a different name and age"}
    )
503
504
505
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
506
        max_completion_tokens=1000,
507
508
        extra_body=dict(structured_outputs={"json": sample_json_schema}),
    )
509
510
511
    message = chat_completion.choices[0].message
    assert message.content is not None
    json2 = json.loads(message.content)
512
    jsonschema.validate(instance=json2, schema=sample_json_schema)
513
514
515
516
517
    assert json1["name"] != json2["name"]
    assert json1["age"] != json2["age"]


@pytest.mark.asyncio
518
519
520
521
async def test_structured_outputs_regex_chat(
    client: openai.AsyncOpenAI,
    sample_regex,
):
522
523
524
525
526
527
528
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": f"Give an example IP address with this regex: {sample_regex}",
        },
    ]
529
530
531
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
532
        max_completion_tokens=20,
533
534
        extra_body=dict(structured_outputs={"regex": sample_regex}),
    )
535
536
    ip1 = chat_completion.choices[0].message.content
    assert ip1 is not None
537
    assert re.fullmatch(sample_regex, ip1) is not None
538
539
540
541
542
543

    messages.append({"role": "assistant", "content": ip1})
    messages.append({"role": "user", "content": "Give me a different one"})
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
544
        max_completion_tokens=20,
545
546
        extra_body=dict(structured_outputs={"regex": sample_regex}),
    )
547
548
    ip2 = chat_completion.choices[0].message.content
    assert ip2 is not None
549
    assert re.fullmatch(sample_regex, ip2) is not None
550
551
552
553
    assert ip1 != ip2


@pytest.mark.asyncio
554
async def test_structured_outputs_type_error(client: openai.AsyncOpenAI):
555
556
557
558
559
560
561
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": "The best language for type-safe systems programming is ",
        },
    ]
562
563

    with pytest.raises(openai.BadRequestError):
564
565
566
        _ = await client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
567
568
            extra_body=dict(structured_outputs={"regex": {1: "Python", 2: "C++"}}),
        )
569
570
571


@pytest.mark.asyncio
572
async def test_structured_outputs_choice_chat_logprobs(
573
574
575
576
577
578
579
580
581
    client: openai.AsyncOpenAI, sample_structured_outputs_choices
):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": "The best language for type-safe systems programming is ",
        },
    ]
582
583
584
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
585
        max_completion_tokens=10,
586
587
        logprobs=True,
        top_logprobs=5,
588
        extra_body=dict(
589
590
591
            structured_outputs={"choice": sample_structured_outputs_choices}
        ),
    )
592
593
594
595
596
597
598
599
600
601
602

    assert chat_completion.choices[0].logprobs is not None
    assert chat_completion.choices[0].logprobs.content is not None
    top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs

    # -9999.0 is the minimum logprob returned by OpenAI
    for item in top_logprobs:
        assert item.logprob >= -9999.0, f"Failed (top_logprobs={top_logprobs})"


@pytest.mark.asyncio
603
604
605
606
async def test_named_tool_use(
    client: openai.AsyncOpenAI,
    sample_json_schema,
):
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": (
                "Give an example JSON for an employee profile using the specified tool."
            ),
        },
    ]
    tools = [
        {
            "type": "function",
            "function": {
                "name": "dummy_function_name",
                "description": "This is a dummy function",
                "parameters": sample_json_schema,
            },
624
        }
625
626
    ]
    tool_choice = {"type": "function", "function": {"name": "dummy_function_name"}}
627
628
629
630
631
632

    # non-streaming

    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
633
        max_completion_tokens=1000,
634
635
        tools=tools,
        tool_choice=tool_choice,
636
    )
637
638
639
640
    message = chat_completion.choices[0].message
    assert len(message.content) == 0
    json_string = message.tool_calls[0].function.arguments
    json1 = json.loads(json_string)
641
    jsonschema.validate(instance=json1, schema=sample_json_schema)
642
643

    messages.append({"role": "assistant", "content": json_string})
644
645
646
    messages.append(
        {"role": "user", "content": "Give me another one with a different name and age"}
    )
647
648
649

    # streaming

650
651
652
653
654
655
656
657
    stream = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
        max_completion_tokens=1000,
        tools=tools,
        tool_choice=tool_choice,
        stream=True,
    )
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672

    output = []
    finish_reason_count = 0
    async for chunk in stream:
        delta = chunk.choices[0].delta
        if delta.role:
            assert delta.role == "assistant"
        assert delta.content is None or len(delta.content) == 0
        if delta.tool_calls:
            output.append(delta.tool_calls[0].function.arguments)
        if chunk.choices[0].finish_reason is not None:
            finish_reason_count += 1
    # finish reason should only return in last block
    assert finish_reason_count == 1
    json2 = json.loads("".join(output))
673
    jsonschema.validate(instance=json2, schema=sample_json_schema)
674
675
676
677
678
    assert json1["name"] != json2["name"]
    assert json1["age"] != json2["age"]


@pytest.mark.asyncio
679
680
681
682
683
684
685
686
687
688
689
async def test_inconsistent_tool_choice_and_tools(
    client: openai.AsyncOpenAI, sample_json_schema
):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": f"Give an example JSON for an employee profile that "
            f"fits this schema: {sample_json_schema}",
        },
    ]
690
691

    with pytest.raises(openai.BadRequestError):
692
693
694
695
696
697
698
699
700
        await client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
            max_completion_tokens=1000,
            tool_choice={
                "type": "function",
                "function": {"name": "dummy_function_name"},
            },
        )
701
702
703
704
705

    with pytest.raises(openai.BadRequestError):
        await client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
706
            max_completion_tokens=1000,
707
708
709
710
711
712
713
714
            tools=[
                {
                    "type": "function",
                    "function": {
                        "name": "dummy_function_name",
                        "description": "This is a dummy function",
                        "parameters": sample_json_schema,
                    },
715
                }
716
            ],
717
718
            tool_choice={
                "type": "function",
719
720
721
                "function": {"name": "nondefined_function_name"},
            },
        )
722
723
724
725
726
    with pytest.raises(openai.BadRequestError):
        await client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
            max_completion_tokens=1000,
727
728
729
730
731
732
733
734
            tools=[
                {
                    "type": "function",
                    "function": {
                        "name": "dummy_function_name",
                        "description": "This is a dummy function",
                        "parameters": sample_json_schema,
                    },
735
                }
736
737
738
            ],
            tool_choice={},
        )
739
740
741
742
743
744
745


@pytest.mark.asyncio
async def test_response_format_json_object(client: openai.AsyncOpenAI):
    for _ in range(2):
        resp = await client.chat.completions.create(
            model=MODEL_NAME,
746
747
748
749
750
751
752
753
754
755
756
            messages=[
                {
                    "role": "user",
                    "content": (
                        "what is 1+1? please respond with a JSON object, "
                        'the format is {"result": 2}'
                    ),
                }
            ],
            response_format={"type": "json_object"},
        )
757
758
759
760
761
762
763
764

        content = resp.choices[0].message.content
        assert content is not None

        loaded = json.loads(content)
        assert loaded == {"result": 2}, loaded


765
@pytest.mark.asyncio
766
async def test_response_format_json_schema(client: openai.AsyncOpenAI):
767
768
    prompt = 'what is 1+1? The format is "result": 2'
    # Check that this prompt cannot lead to a valid JSON without json_schema
769
770
771
    for _ in range(2):
        resp = await client.chat.completions.create(
            model=MODEL_NAME,
772
            messages=[{"role": "user", "content": prompt}],
773
774
775
776
777
778
779
780
781
782
        )
        content = resp.choices[0].message.content
        assert content is not None
        with pytest.raises((json.JSONDecodeError, AssertionError)):
            loaded = json.loads(content)
            assert loaded == {"result": 2}, loaded

    for _ in range(2):
        resp = await client.chat.completions.create(
            model=MODEL_NAME,
783
            messages=[{"role": "user", "content": prompt}],
784
785
786
787
788
789
790
            response_format={
                "type": "json_schema",
                "json_schema": {
                    "name": "foo_test",
                    "schema": {
                        "type": "object",
                        "properties": {
791
                            "result": {"type": "integer"},
792
793
                        },
                    },
794
795
796
                },
            },
        )
797
798
799
800
801
802
803
804

        content = resp.choices[0].message.content
        assert content is not None

        loaded = json.loads(content)
        assert loaded == {"result": 2}, loaded


805
@pytest.mark.asyncio
806
807
808
async def test_extra_fields_allowed(client: openai.AsyncOpenAI):
    resp = await client.chat.completions.create(
        model=MODEL_NAME,
809
810
811
812
813
814
815
        messages=[
            {
                "role": "user",
                "content": "what is 1+1?",
                "extra_field": "0",
            }
        ],  # type: ignore
816
        temperature=0,
817
818
        seed=0,
    )
819
820
821

    content = resp.choices[0].message.content
    assert content is not None
822
823
824
825


@pytest.mark.asyncio
async def test_complex_message_content(client: openai.AsyncOpenAI):
826
827
828
829
830
831
    content = [
        {
            "type": "text",
            "text": "what is 1+1? please provide the result without any other text.",
        }
    ]
832
833
    resp = await client.chat.completions.create(
        model=MODEL_NAME,
834
835
836
        messages=[
            {
                "role": "user",
837
                "content": content,
838
839
            }
        ],
840
        temperature=0,
841
842
        seed=0,
    )
843
844
845
846
847
848
849
850
851
852
853
    content = resp.choices[0].message.content
    assert content == "2"


@pytest.mark.asyncio
async def test_custom_role(client: openai.AsyncOpenAI):
    # Not sure how the model handles custom roles so we just check that
    # both string and complex message content are handled in the same way

    resp1 = await client.chat.completions.create(
        model=MODEL_NAME,
854
855
856
857
858
859
        messages=[
            {
                "role": "my-custom-role",
                "content": "what is 1+1?",
            }
        ],  # type: ignore
860
        temperature=0,
861
862
        seed=0,
    )
863
864
865

    resp2 = await client.chat.completions.create(
        model=MODEL_NAME,
866
867
868
869
870
871
        messages=[
            {
                "role": "my-custom-role",
                "content": [{"type": "text", "text": "what is 1+1?"}],
            }
        ],  # type: ignore
872
        temperature=0,
873
874
        seed=0,
    )
875
876
877
878
879
880
881
882

    content1 = resp1.choices[0].message.content
    content2 = resp2.choices[0].message.content
    assert content1 == content2


@pytest.mark.asyncio
async def test_long_seed(client: openai.AsyncOpenAI):
883
    for seed in [torch.iinfo(torch.long).min - 1, torch.iinfo(torch.long).max + 1]:
884
885
886
        with pytest.raises(BadRequestError) as exc_info:
            await client.chat.completions.create(
                model=MODEL_NAME,
887
888
889
890
891
892
                messages=[
                    {
                        "role": "system",
                        "content": "You are a helpful assistant.",
                    }
                ],
893
                temperature=0,
894
895
                seed=seed,
            )
896

897
898
899
900
        assert (
            "greater_than_equal" in exc_info.value.message
            or "less_than_equal" in exc_info.value.message
        )
901
902


903
@pytest.mark.asyncio
904
905
906
907
908
async def test_invocations(server: RemoteOpenAIServer, client: openai.AsyncOpenAI):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
909
910
911
912
913
914
915
916
917
918
919

    request_args = {
        "model": MODEL_NAME,
        "messages": messages,
        "max_completion_tokens": 5,
        "temperature": 0.0,
        "logprobs": False,
    }

    chat_completion = await client.chat.completions.create(**request_args)

920
921
922
    invocation_response = requests.post(
        server.url_for("invocations"), json=request_args
    )
923
924
925
926
927
928
929
    invocation_response.raise_for_status()

    chat_output = chat_completion.model_dump()
    invocation_output = invocation_response.json()

    assert chat_output.keys() == invocation_output.keys()
    assert chat_output["choices"] == invocation_output["choices"]