"vscode:/vscode.git/clone" did not exist on "cd7740ac5c3906b2913d58ade61f231ec3a93296"
test_chat.py 28.9 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
# imports for structured outputs tests
5
import json
6
from typing import Optional
7
8
9
10

import jsonschema
import openai  # use the official client for correctness check
import pytest
11
import pytest_asyncio
12
import regex as re
13
import requests
14
import torch
15
from openai import BadRequestError
16

17
from ...utils import RemoteOpenAIServer
18
19
20
21
22

# any model with a chat template should work here
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"


23
@pytest.fixture(scope="module")
24
25
def monkeypatch_module():
    from _pytest.monkeypatch import MonkeyPatch
26

27
28
29
30
31
    mpatch = MonkeyPatch()
    yield mpatch
    mpatch.undo()


32
@pytest.fixture(scope="module")
33
34
def server(monkeypatch_module, zephyr_lora_files):  # noqa: F811
    monkeypatch_module.setenv("VLLM_USE_V1", "1")
35

36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
    args = [
        # use half precision for speed and memory savings in CI environment
        "--dtype",
        "bfloat16",
        "--max-model-len",
        "8192",
        "--enforce-eager",
        # lora config below
        "--enable-lora",
        "--lora-modules",
        f"zephyr-lora={zephyr_lora_files}",
        "--max-lora-rank",
        "64",
        "--max-cpu-loras",
        "2",
        "--max-num-seqs",
        "128",
    ]

    with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
56
        yield remote_server
57
58


59
60
61
62
@pytest_asyncio.fixture
async def client(server):
    async with server.get_async_client() as async_client:
        yield async_client
63
64
65
66
67
68


@pytest.mark.asyncio
@pytest.mark.parametrize(
    # first test base model, then test loras
    "model_name",
69
    [MODEL_NAME, "zephyr-lora"],
70
71
)
async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
72
73
74
75
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
76

77
78
79
80
81
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=5,
        temperature=0.0,
82
83
        logprobs=False,
    )
84
85
86
87
88
89
90
91
92
93
94
95

    choice = chat_completion.choices[0]
    assert choice.logprobs is None


@pytest.mark.asyncio
@pytest.mark.parametrize(
    # just test 1 lora hereafter
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
96
97
98
99
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
100

101
102
103
104
105
106
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=5,
        temperature=0.0,
        logprobs=True,
107
108
        top_logprobs=0,
    )
109
110
111
112
113
114
115
116
117
118
119
120
121

    choice = chat_completion.choices[0]
    assert choice.logprobs is not None
    assert choice.logprobs.content is not None
    assert len(choice.logprobs.content[0].top_logprobs) == 0


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
122
123
124
125
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
126

127
128
129
130
131
132
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=5,
        temperature=0.0,
        logprobs=True,
133
134
        top_logprobs=5,
    )
135
136
137
138
139
140
141
142
143
144
145
146

    choice = chat_completion.choices[0]
    assert choice.logprobs is not None
    assert choice.logprobs.content is not None
    assert len(choice.logprobs.content[0].top_logprobs) == 5


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
147
148
149
150
151
async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI, model_name: str):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
152
153
154

    # Default max_logprobs is 20, so this should raise an error
    with pytest.raises((openai.BadRequestError, openai.APIError)):
155
156
157
158
159
160
161
162
        stream = await client.chat.completions.create(
            model=model_name,
            messages=messages,
            max_completion_tokens=10,
            logprobs=True,
            top_logprobs=21,
            stream=True,
        )
163
164
165
166
        async for chunk in stream:
            ...

    with pytest.raises(openai.BadRequestError):
167
168
169
170
171
172
173
174
        await client.chat.completions.create(
            model=model_name,
            messages=messages,
            max_completion_tokens=10,
            logprobs=True,
            top_logprobs=30,
            stream=False,
        )
175
176

    # the server should still work afterwards
177
    chat_completion = await client.chat.completions.create(
178
179
        model=model_name, messages=messages, max_completion_tokens=10, stream=False
    )
180
181
182
183
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 0


184
185
186
187
188
@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name, prompt_logprobs",
    [(MODEL_NAME, 1), (MODEL_NAME, 0), (MODEL_NAME, -1), (MODEL_NAME, None)],
)
189
190
191
async def test_prompt_logprobs_chat(
    client: openai.AsyncOpenAI, model_name: str, prompt_logprobs: Optional[int]
):
192
    params: dict = {
193
194
195
196
197
198
199
200
201
202
        "messages": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Who won the world series in 2020?"},
            {
                "role": "assistant",
                "content": "The Los Angeles Dodgers won the World Series in 2020.",
            },
            {"role": "user", "content": "Where was it played?"},
        ],
        "model": model_name,
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
    }

    if prompt_logprobs is not None:
        params["extra_body"] = {"prompt_logprobs": prompt_logprobs}

    if prompt_logprobs is not None and prompt_logprobs < 0:
        with pytest.raises(BadRequestError):
            await client.chat.completions.create(**params)
    else:
        completion = await client.chat.completions.create(**params)
        if prompt_logprobs is not None:
            assert completion.prompt_logprobs is not None
            assert len(completion.prompt_logprobs) > 0
        else:
            assert completion.prompt_logprobs is None


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME],
)
225
226
227
async def test_more_than_one_prompt_logprobs_chat(
    client: openai.AsyncOpenAI, model_name: str
):
228
    params: dict = {
229
230
231
232
233
234
235
236
237
238
239
        "messages": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Who won the world series in 2020?"},
            {
                "role": "assistant",
                "content": "The Los Angeles Dodgers won the World Series in 2020.",
            },
            {"role": "user", "content": "Where was it played?"},
        ],
        "model": model_name,
        "extra_body": {"prompt_logprobs": 1},
240
241
242
243
244
245
246
247
248
249
250
    }

    completion_1 = await client.chat.completions.create(**params)

    params["extra_body"] = {"prompt_logprobs": 2}
    completion_2 = await client.chat.completions.create(**params)

    assert len(completion_1.prompt_logprobs[3]) == 1
    assert len(completion_2.prompt_logprobs[3]) == 2


251
252
253
254
255
@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
256
257
258
259
260
async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
261
262

    # test single completion
263
264
265
266
267
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
        logprobs=True,
268
269
        top_logprobs=5,
    )
270
271
272
273
274
275
    assert chat_completion.id is not None
    assert len(chat_completion.choices) == 1

    choice = chat_completion.choices[0]
    assert choice.finish_reason == "length"
    assert chat_completion.usage == openai.types.CompletionUsage(
276
277
        completion_tokens=10, prompt_tokens=37, total_tokens=47
    )
278
279
280
281
282
283
284
285
286
287
288

    message = choice.message
    assert message.content is not None and len(message.content) >= 10
    assert message.role == "assistant"
    messages.append({"role": "assistant", "content": message.content})

    # test multi-turn dialogue
    messages.append({"role": "user", "content": "express your result in json"})
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
289
        max_completion_tokens=10,
290
291
292
293
294
295
296
297
298
299
300
301
    )
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 0


@pytest.mark.asyncio
@pytest.mark.parametrize(
    # just test 1 lora hereafter
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
302
303
304
305
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
306
307
308
309
310

    # test single completion
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
311
        max_completion_tokens=10,
312
313
314
315
316
317
318
319
320
        temperature=0.0,
    )
    output = chat_completion.choices[0].message.content
    stop_reason = chat_completion.choices[0].finish_reason

    # test streaming
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
321
        max_completion_tokens=10,
322
323
324
        temperature=0.0,
        stream=True,
    )
325
    chunks: list[str] = []
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
    finish_reason_count = 0
    async for chunk in stream:
        delta = chunk.choices[0].delta
        if delta.role:
            assert delta.role == "assistant"
        if delta.content:
            chunks.append(delta.content)
        if chunk.choices[0].finish_reason is not None:
            finish_reason_count += 1
    # finish reason should only return in last block
    assert finish_reason_count == 1
    assert chunk.choices[0].finish_reason == stop_reason
    assert delta.content
    assert "".join(chunks) == output


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
)
347
348
349
350
351
352
353
async def test_chat_completion_stream_options(
    client: openai.AsyncOpenAI, model_name: str
):
    messages = [
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "What is the capital of France?"},
    ]
354
355
356
357
358

    # Test stream=True, stream_options={"include_usage": False}
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
359
        max_completion_tokens=10,
360
361
        temperature=0.0,
        stream=True,
362
363
        stream_options={"include_usage": False},
    )
364
365
366
    async for chunk in stream:
        assert chunk.usage is None

367
368
    # Test stream=True, stream_options={"include_usage": True,
    #                                   "continuous_usage_stats": False}}
369
370
371
372
373
374
375
376
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
        temperature=0.0,
        stream=True,
        stream_options={"include_usage": True, "continuous_usage_stats": False},
    )
377
378
379
380
381
382
383
384
385
386
387

    async for chunk in stream:
        if chunk.choices[0].finish_reason is None:
            assert chunk.usage is None
        else:
            assert chunk.usage is None
            final_chunk = await stream.__anext__()
            assert final_chunk.usage is not None
            assert final_chunk.usage.prompt_tokens > 0
            assert final_chunk.usage.completion_tokens > 0
            assert final_chunk.usage.total_tokens == (
388
389
                final_chunk.usage.prompt_tokens + final_chunk.usage.completion_tokens
            )
390
391
392
393
394
395
396
            assert final_chunk.choices == []

    # Test stream=False, stream_options={"include_usage": None}
    with pytest.raises(BadRequestError):
        await client.chat.completions.create(
            model=model_name,
            messages=messages,
397
            max_completion_tokens=10,
398
399
            temperature=0.0,
            stream=False,
400
401
            stream_options={"include_usage": None},
        )
402
403
404
405
406
407

    # Test stream=False, stream_options={"include_usage": True}
    with pytest.raises(BadRequestError):
        await client.chat.completions.create(
            model=model_name,
            messages=messages,
408
            max_completion_tokens=10,
409
410
            temperature=0.0,
            stream=False,
411
412
            stream_options={"include_usage": True},
        )
413

414
415
416
417
418
    # Test stream=True, stream_options={"include_usage": True,
    #                           "continuous_usage_stats": True}
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
419
        max_completion_tokens=10,
420
        extra_body=dict(min_tokens=10),
421
422
423
424
        temperature=0.0,
        stream=True,
        stream_options={
            "include_usage": True,
425
            "continuous_usage_stats": True,
426
427
        },
    )
428
    last_completion_tokens = 0
429
430
    async for chunk in stream:
        assert chunk.usage.prompt_tokens >= 0
431
432
433
434
435
436
437
438
439
440
441
        assert (
            last_completion_tokens == 0
            or chunk.usage.completion_tokens > last_completion_tokens
            or (
                not chunk.choices
                and chunk.usage.completion_tokens == last_completion_tokens
            )
        )
        assert chunk.usage.total_tokens == (
            chunk.usage.prompt_tokens + chunk.usage.completion_tokens
        )
442
443
444
        last_completion_tokens = chunk.usage.completion_tokens

    assert last_completion_tokens == 10
445

446
447

@pytest.mark.asyncio
448
async def test_structured_outputs_choice_chat(
449
450
451
    client: openai.AsyncOpenAI,
    sample_structured_outputs_choices,
):
452
453
454
455
456
457
458
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": "The best language for type-safe systems programming is ",
        },
    ]
459
460
461
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
462
        max_completion_tokens=10,
463
        temperature=0.7,
464
        extra_body=dict(
465
466
467
            structured_outputs={"choice": sample_structured_outputs_choices}
        ),
    )
468
    choice1 = chat_completion.choices[0].message.content
469
    assert choice1 in sample_structured_outputs_choices
470
471

    messages.append({"role": "assistant", "content": choice1})
472
    messages.append({"role": "user", "content": "I disagree, pick another one"})
473
474
475
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
476
        max_completion_tokens=10,
477
        temperature=0.7,
478
        extra_body=dict(
479
480
481
            structured_outputs={"choice": sample_structured_outputs_choices}
        ),
    )
482
    choice2 = chat_completion.choices[0].message.content
483
    assert choice2 in sample_structured_outputs_choices
484
485
486
487
    assert choice1 != choice2


@pytest.mark.asyncio
488
489
490
491
async def test_structured_outputs_json_chat(
    client: openai.AsyncOpenAI,
    sample_json_schema,
):
492
493
494
495
496
497
498
499
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": f"Give an example JSON for an employee profile that "
            f"fits this schema: {sample_json_schema}",
        },
    ]
500
501
502
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
503
        max_completion_tokens=1000,
504
505
        extra_body=dict(structured_outputs={"json": sample_json_schema}),
    )
506
507
508
    message = chat_completion.choices[0].message
    assert message.content is not None
    json1 = json.loads(message.content)
509
    jsonschema.validate(instance=json1, schema=sample_json_schema)
510
511

    messages.append({"role": "assistant", "content": message.content})
512
513
514
    messages.append(
        {"role": "user", "content": "Give me another one with a different name and age"}
    )
515
516
517
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
518
        max_completion_tokens=1000,
519
520
        extra_body=dict(structured_outputs={"json": sample_json_schema}),
    )
521
522
523
    message = chat_completion.choices[0].message
    assert message.content is not None
    json2 = json.loads(message.content)
524
    jsonschema.validate(instance=json2, schema=sample_json_schema)
525
526
527
528
529
    assert json1["name"] != json2["name"]
    assert json1["age"] != json2["age"]


@pytest.mark.asyncio
530
531
532
533
async def test_structured_outputs_regex_chat(
    client: openai.AsyncOpenAI,
    sample_regex,
):
534
535
536
537
538
539
540
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": f"Give an example IP address with this regex: {sample_regex}",
        },
    ]
541
542
543
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
544
        max_completion_tokens=20,
545
546
        extra_body=dict(structured_outputs={"regex": sample_regex}),
    )
547
548
    ip1 = chat_completion.choices[0].message.content
    assert ip1 is not None
549
    assert re.fullmatch(sample_regex, ip1) is not None
550
551
552
553
554
555

    messages.append({"role": "assistant", "content": ip1})
    messages.append({"role": "user", "content": "Give me a different one"})
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
556
        max_completion_tokens=20,
557
558
        extra_body=dict(structured_outputs={"regex": sample_regex}),
    )
559
560
    ip2 = chat_completion.choices[0].message.content
    assert ip2 is not None
561
    assert re.fullmatch(sample_regex, ip2) is not None
562
563
564
565
    assert ip1 != ip2


@pytest.mark.asyncio
566
async def test_structured_outputs_type_error(client: openai.AsyncOpenAI):
567
568
569
570
571
572
573
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": "The best language for type-safe systems programming is ",
        },
    ]
574
575

    with pytest.raises(openai.BadRequestError):
576
577
578
        _ = await client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
579
580
            extra_body=dict(structured_outputs={"regex": {1: "Python", 2: "C++"}}),
        )
581
582
583


@pytest.mark.asyncio
584
async def test_structured_outputs_choice_chat_logprobs(
585
586
587
588
589
590
591
592
593
    client: openai.AsyncOpenAI, sample_structured_outputs_choices
):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": "The best language for type-safe systems programming is ",
        },
    ]
594
595
596
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
597
        max_completion_tokens=10,
598
599
        logprobs=True,
        top_logprobs=5,
600
        extra_body=dict(
601
602
603
            structured_outputs={"choice": sample_structured_outputs_choices}
        ),
    )
604
605
606
607
608
609
610
611
612
613
614

    assert chat_completion.choices[0].logprobs is not None
    assert chat_completion.choices[0].logprobs.content is not None
    top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs

    # -9999.0 is the minimum logprob returned by OpenAI
    for item in top_logprobs:
        assert item.logprob >= -9999.0, f"Failed (top_logprobs={top_logprobs})"


@pytest.mark.asyncio
615
616
617
618
async def test_named_tool_use(
    client: openai.AsyncOpenAI,
    sample_json_schema,
):
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": (
                "Give an example JSON for an employee profile using the specified tool."
            ),
        },
    ]
    tools = [
        {
            "type": "function",
            "function": {
                "name": "dummy_function_name",
                "description": "This is a dummy function",
                "parameters": sample_json_schema,
            },
636
        }
637
638
    ]
    tool_choice = {"type": "function", "function": {"name": "dummy_function_name"}}
639
640
641
642
643
644

    # non-streaming

    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
645
        max_completion_tokens=1000,
646
647
        tools=tools,
        tool_choice=tool_choice,
648
    )
649
650
651
652
    message = chat_completion.choices[0].message
    assert len(message.content) == 0
    json_string = message.tool_calls[0].function.arguments
    json1 = json.loads(json_string)
653
    jsonschema.validate(instance=json1, schema=sample_json_schema)
654
655

    messages.append({"role": "assistant", "content": json_string})
656
657
658
    messages.append(
        {"role": "user", "content": "Give me another one with a different name and age"}
    )
659
660
661

    # streaming

662
663
664
665
666
667
668
669
    stream = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
        max_completion_tokens=1000,
        tools=tools,
        tool_choice=tool_choice,
        stream=True,
    )
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684

    output = []
    finish_reason_count = 0
    async for chunk in stream:
        delta = chunk.choices[0].delta
        if delta.role:
            assert delta.role == "assistant"
        assert delta.content is None or len(delta.content) == 0
        if delta.tool_calls:
            output.append(delta.tool_calls[0].function.arguments)
        if chunk.choices[0].finish_reason is not None:
            finish_reason_count += 1
    # finish reason should only return in last block
    assert finish_reason_count == 1
    json2 = json.loads("".join(output))
685
    jsonschema.validate(instance=json2, schema=sample_json_schema)
686
687
688
689
690
    assert json1["name"] != json2["name"]
    assert json1["age"] != json2["age"]


@pytest.mark.asyncio
691
692
693
694
695
696
697
698
699
700
701
async def test_inconsistent_tool_choice_and_tools(
    client: openai.AsyncOpenAI, sample_json_schema
):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": f"Give an example JSON for an employee profile that "
            f"fits this schema: {sample_json_schema}",
        },
    ]
702
703

    with pytest.raises(openai.BadRequestError):
704
705
706
707
708
709
710
711
712
        await client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
            max_completion_tokens=1000,
            tool_choice={
                "type": "function",
                "function": {"name": "dummy_function_name"},
            },
        )
713
714
715
716
717

    with pytest.raises(openai.BadRequestError):
        await client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
718
            max_completion_tokens=1000,
719
720
721
722
723
724
725
726
            tools=[
                {
                    "type": "function",
                    "function": {
                        "name": "dummy_function_name",
                        "description": "This is a dummy function",
                        "parameters": sample_json_schema,
                    },
727
                }
728
            ],
729
730
            tool_choice={
                "type": "function",
731
732
733
                "function": {"name": "nondefined_function_name"},
            },
        )
734
735
736
737
738
    with pytest.raises(openai.BadRequestError):
        await client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
            max_completion_tokens=1000,
739
740
741
742
743
744
745
746
            tools=[
                {
                    "type": "function",
                    "function": {
                        "name": "dummy_function_name",
                        "description": "This is a dummy function",
                        "parameters": sample_json_schema,
                    },
747
                }
748
749
750
            ],
            tool_choice={},
        )
751
752
753
754
755
756
757


@pytest.mark.asyncio
async def test_response_format_json_object(client: openai.AsyncOpenAI):
    for _ in range(2):
        resp = await client.chat.completions.create(
            model=MODEL_NAME,
758
759
760
761
762
763
764
765
766
767
768
            messages=[
                {
                    "role": "user",
                    "content": (
                        "what is 1+1? please respond with a JSON object, "
                        'the format is {"result": 2}'
                    ),
                }
            ],
            response_format={"type": "json_object"},
        )
769
770
771
772
773
774
775
776

        content = resp.choices[0].message.content
        assert content is not None

        loaded = json.loads(content)
        assert loaded == {"result": 2}, loaded


777
@pytest.mark.asyncio
778
async def test_response_format_json_schema(client: openai.AsyncOpenAI):
779
780
    prompt = 'what is 1+1? The format is "result": 2'
    # Check that this prompt cannot lead to a valid JSON without json_schema
781
782
783
    for _ in range(2):
        resp = await client.chat.completions.create(
            model=MODEL_NAME,
784
            messages=[{"role": "user", "content": prompt}],
785
786
787
788
789
790
791
792
793
794
        )
        content = resp.choices[0].message.content
        assert content is not None
        with pytest.raises((json.JSONDecodeError, AssertionError)):
            loaded = json.loads(content)
            assert loaded == {"result": 2}, loaded

    for _ in range(2):
        resp = await client.chat.completions.create(
            model=MODEL_NAME,
795
            messages=[{"role": "user", "content": prompt}],
796
797
798
799
800
801
802
            response_format={
                "type": "json_schema",
                "json_schema": {
                    "name": "foo_test",
                    "schema": {
                        "type": "object",
                        "properties": {
803
                            "result": {"type": "integer"},
804
805
                        },
                    },
806
807
808
                },
            },
        )
809
810
811
812
813
814
815
816

        content = resp.choices[0].message.content
        assert content is not None

        loaded = json.loads(content)
        assert loaded == {"result": 2}, loaded


817
@pytest.mark.asyncio
818
819
820
async def test_extra_fields_allowed(client: openai.AsyncOpenAI):
    resp = await client.chat.completions.create(
        model=MODEL_NAME,
821
822
823
824
825
826
827
        messages=[
            {
                "role": "user",
                "content": "what is 1+1?",
                "extra_field": "0",
            }
        ],  # type: ignore
828
        temperature=0,
829
830
        seed=0,
    )
831
832
833

    content = resp.choices[0].message.content
    assert content is not None
834
835
836
837


@pytest.mark.asyncio
async def test_complex_message_content(client: openai.AsyncOpenAI):
838
839
840
841
842
843
    content = [
        {
            "type": "text",
            "text": "what is 1+1? please provide the result without any other text.",
        }
    ]
844
845
    resp = await client.chat.completions.create(
        model=MODEL_NAME,
846
847
848
        messages=[
            {
                "role": "user",
849
                "content": content,
850
851
            }
        ],
852
        temperature=0,
853
854
        seed=0,
    )
855
856
857
858
859
860
861
862
863
864
865
    content = resp.choices[0].message.content
    assert content == "2"


@pytest.mark.asyncio
async def test_custom_role(client: openai.AsyncOpenAI):
    # Not sure how the model handles custom roles so we just check that
    # both string and complex message content are handled in the same way

    resp1 = await client.chat.completions.create(
        model=MODEL_NAME,
866
867
868
869
870
871
        messages=[
            {
                "role": "my-custom-role",
                "content": "what is 1+1?",
            }
        ],  # type: ignore
872
        temperature=0,
873
874
        seed=0,
    )
875
876
877

    resp2 = await client.chat.completions.create(
        model=MODEL_NAME,
878
879
880
881
882
883
        messages=[
            {
                "role": "my-custom-role",
                "content": [{"type": "text", "text": "what is 1+1?"}],
            }
        ],  # type: ignore
884
        temperature=0,
885
886
        seed=0,
    )
887
888
889
890
891
892
893
894

    content1 = resp1.choices[0].message.content
    content2 = resp2.choices[0].message.content
    assert content1 == content2


@pytest.mark.asyncio
async def test_long_seed(client: openai.AsyncOpenAI):
895
    for seed in [torch.iinfo(torch.long).min - 1, torch.iinfo(torch.long).max + 1]:
896
897
898
        with pytest.raises(BadRequestError) as exc_info:
            await client.chat.completions.create(
                model=MODEL_NAME,
899
900
901
902
903
904
                messages=[
                    {
                        "role": "system",
                        "content": "You are a helpful assistant.",
                    }
                ],
905
                temperature=0,
906
907
                seed=seed,
            )
908

909
910
911
912
        assert (
            "greater_than_equal" in exc_info.value.message
            or "less_than_equal" in exc_info.value.message
        )
913
914


915
@pytest.mark.asyncio
916
917
918
919
920
async def test_invocations(server: RemoteOpenAIServer, client: openai.AsyncOpenAI):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
921
922
923
924
925
926
927
928
929
930
931

    request_args = {
        "model": MODEL_NAME,
        "messages": messages,
        "max_completion_tokens": 5,
        "temperature": 0.0,
        "logprobs": False,
    }

    chat_completion = await client.chat.completions.create(**request_args)

932
933
934
    invocation_response = requests.post(
        server.url_for("invocations"), json=request_args
    )
935
936
937
938
939
940
941
    invocation_response.raise_for_status()

    chat_output = chat_completion.model_dump()
    invocation_output = invocation_response.json()

    assert chat_output.keys() == invocation_output.keys()
    assert chat_output["choices"] == invocation_output["choices"]