test_chat.py 24.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
# imports for structured outputs tests
5
6
7
8
9
import json

import jsonschema
import openai  # use the official client for correctness check
import pytest
10
import pytest_asyncio
11
import regex as re
12
import requests
13
import torch
14
from openai import BadRequestError
15

16
from ...utils import RemoteOpenAIServer
17
18
19
20
21

# any model with a chat template should work here
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"


22
@pytest.fixture(scope="module")
23
def server(zephyr_lora_files):  # noqa: F811
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
    args = [
        # use half precision for speed and memory savings in CI environment
        "--dtype",
        "bfloat16",
        "--max-model-len",
        "8192",
        "--enforce-eager",
        # lora config below
        "--enable-lora",
        "--lora-modules",
        f"zephyr-lora={zephyr_lora_files}",
        "--max-lora-rank",
        "64",
        "--max-cpu-loras",
        "2",
        "--max-num-seqs",
        "128",
    ]

    with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
44
        yield remote_server
45
46


47
48
49
50
@pytest_asyncio.fixture
async def client(server):
    async with server.get_async_client() as async_client:
        yield async_client
51
52
53
54
55
56


@pytest.mark.asyncio
@pytest.mark.parametrize(
    # first test base model, then test loras
    "model_name",
57
    [MODEL_NAME, "zephyr-lora"],
58
59
)
async def test_no_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
60
61
62
63
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
64

65
66
67
68
69
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=5,
        temperature=0.0,
70
71
        logprobs=False,
    )
72
73
74
75
76
77
78
79
80
81
82
83

    choice = chat_completion.choices[0]
    assert choice.logprobs is None


@pytest.mark.asyncio
@pytest.mark.parametrize(
    # just test 1 lora hereafter
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
async def test_zero_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
84
85
86
87
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
88

89
90
91
92
93
94
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=5,
        temperature=0.0,
        logprobs=True,
95
96
        top_logprobs=0,
    )
97
98
99
100
101
102
103
104
105
106
107
108
109

    choice = chat_completion.choices[0]
    assert choice.logprobs is not None
    assert choice.logprobs.content is not None
    assert len(choice.logprobs.content[0].top_logprobs) == 0


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
async def test_some_logprobs_chat(client: openai.AsyncOpenAI, model_name: str):
110
111
112
113
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
114

115
116
117
118
119
120
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=5,
        temperature=0.0,
        logprobs=True,
121
122
        top_logprobs=5,
    )
123
124
125
126
127
128
129
130
131
132
133
134

    choice = chat_completion.choices[0]
    assert choice.logprobs is not None
    assert choice.logprobs.content is not None
    assert len(choice.logprobs.content[0].top_logprobs) == 5


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
135
136
137
138
139
async def test_too_many_chat_logprobs(client: openai.AsyncOpenAI, model_name: str):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
140
141
142

    # Default max_logprobs is 20, so this should raise an error
    with pytest.raises((openai.BadRequestError, openai.APIError)):
143
144
145
146
147
148
149
150
        stream = await client.chat.completions.create(
            model=model_name,
            messages=messages,
            max_completion_tokens=10,
            logprobs=True,
            top_logprobs=21,
            stream=True,
        )
151
152
153
154
        async for chunk in stream:
            ...

    with pytest.raises(openai.BadRequestError):
155
156
157
158
159
160
161
162
        await client.chat.completions.create(
            model=model_name,
            messages=messages,
            max_completion_tokens=10,
            logprobs=True,
            top_logprobs=30,
            stream=False,
        )
163
164

    # the server should still work afterwards
165
    chat_completion = await client.chat.completions.create(
166
167
        model=model_name, messages=messages, max_completion_tokens=10, stream=False
    )
168
169
170
171
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 0


172
173
174
175
176
@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name, prompt_logprobs",
    [(MODEL_NAME, 1), (MODEL_NAME, 0), (MODEL_NAME, -1), (MODEL_NAME, None)],
)
177
async def test_prompt_logprobs_chat(
178
    client: openai.AsyncOpenAI, model_name: str, prompt_logprobs: int | None
179
):
180
    params: dict = {
181
182
183
184
185
186
187
188
189
190
        "messages": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Who won the world series in 2020?"},
            {
                "role": "assistant",
                "content": "The Los Angeles Dodgers won the World Series in 2020.",
            },
            {"role": "user", "content": "Where was it played?"},
        ],
        "model": model_name,
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
    }

    if prompt_logprobs is not None:
        params["extra_body"] = {"prompt_logprobs": prompt_logprobs}

    if prompt_logprobs is not None and prompt_logprobs < 0:
        with pytest.raises(BadRequestError):
            await client.chat.completions.create(**params)
    else:
        completion = await client.chat.completions.create(**params)
        if prompt_logprobs is not None:
            assert completion.prompt_logprobs is not None
            assert len(completion.prompt_logprobs) > 0
        else:
            assert completion.prompt_logprobs is None


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME],
)
213
214
215
async def test_more_than_one_prompt_logprobs_chat(
    client: openai.AsyncOpenAI, model_name: str
):
216
    params: dict = {
217
218
219
220
221
222
223
224
225
226
227
        "messages": [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Who won the world series in 2020?"},
            {
                "role": "assistant",
                "content": "The Los Angeles Dodgers won the World Series in 2020.",
            },
            {"role": "user", "content": "Where was it played?"},
        ],
        "model": model_name,
        "extra_body": {"prompt_logprobs": 1},
228
229
230
231
232
233
234
235
236
237
238
    }

    completion_1 = await client.chat.completions.create(**params)

    params["extra_body"] = {"prompt_logprobs": 2}
    completion_2 = await client.chat.completions.create(**params)

    assert len(completion_1.prompt_logprobs[3]) == 1
    assert len(completion_2.prompt_logprobs[3]) == 2


239
240
241
242
243
@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
244
245
246
247
248
async def test_single_chat_session(client: openai.AsyncOpenAI, model_name: str):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
249
250

    # test single completion
251
252
253
254
255
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
        logprobs=True,
256
257
        top_logprobs=5,
    )
258
259
260
261
262
263
    assert chat_completion.id is not None
    assert len(chat_completion.choices) == 1

    choice = chat_completion.choices[0]
    assert choice.finish_reason == "length"
    assert chat_completion.usage == openai.types.CompletionUsage(
264
265
        completion_tokens=10, prompt_tokens=37, total_tokens=47
    )
266
267
268
269
270
271
272
273
274
275
276

    message = choice.message
    assert message.content is not None and len(message.content) >= 10
    assert message.role == "assistant"
    messages.append({"role": "assistant", "content": message.content})

    # test multi-turn dialogue
    messages.append({"role": "user", "content": "express your result in json"})
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
277
        max_completion_tokens=10,
278
279
280
281
282
283
284
285
286
287
288
289
    )
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 0


@pytest.mark.asyncio
@pytest.mark.parametrize(
    # just test 1 lora hereafter
    "model_name",
    [MODEL_NAME, "zephyr-lora"],
)
async def test_chat_streaming(client: openai.AsyncOpenAI, model_name: str):
290
291
292
293
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
294
295
296
297
298

    # test single completion
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
299
        max_completion_tokens=10,
300
301
302
303
304
305
306
307
308
        temperature=0.0,
    )
    output = chat_completion.choices[0].message.content
    stop_reason = chat_completion.choices[0].finish_reason

    # test streaming
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
309
        max_completion_tokens=10,
310
311
312
        temperature=0.0,
        stream=True,
    )
313
    chunks: list[str] = []
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
    finish_reason_count = 0
    async for chunk in stream:
        delta = chunk.choices[0].delta
        if delta.role:
            assert delta.role == "assistant"
        if delta.content:
            chunks.append(delta.content)
        if chunk.choices[0].finish_reason is not None:
            finish_reason_count += 1
    # finish reason should only return in last block
    assert finish_reason_count == 1
    assert chunk.choices[0].finish_reason == stop_reason
    assert delta.content
    assert "".join(chunks) == output


@pytest.mark.asyncio
@pytest.mark.parametrize(
    "model_name",
    ["HuggingFaceH4/zephyr-7b-beta", "zephyr-lora"],
)
335
336
337
338
339
340
341
async def test_chat_completion_stream_options(
    client: openai.AsyncOpenAI, model_name: str
):
    messages = [
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "What is the capital of France?"},
    ]
342
343
344
345
346

    # Test stream=True, stream_options={"include_usage": False}
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
347
        max_completion_tokens=10,
348
349
        temperature=0.0,
        stream=True,
350
351
        stream_options={"include_usage": False},
    )
352
353
354
    async for chunk in stream:
        assert chunk.usage is None

355
356
    # Test stream=True, stream_options={"include_usage": True,
    #                                   "continuous_usage_stats": False}}
357
358
359
360
361
362
363
364
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
        temperature=0.0,
        stream=True,
        stream_options={"include_usage": True, "continuous_usage_stats": False},
    )
365
366
367
368
369
370

    async for chunk in stream:
        if chunk.choices[0].finish_reason is None:
            assert chunk.usage is None
        else:
            assert chunk.usage is None
371
            final_chunk = await anext(stream)
372
373
374
375
            assert final_chunk.usage is not None
            assert final_chunk.usage.prompt_tokens > 0
            assert final_chunk.usage.completion_tokens > 0
            assert final_chunk.usage.total_tokens == (
376
377
                final_chunk.usage.prompt_tokens + final_chunk.usage.completion_tokens
            )
378
379
380
381
382
383
384
            assert final_chunk.choices == []

    # Test stream=False, stream_options={"include_usage": None}
    with pytest.raises(BadRequestError):
        await client.chat.completions.create(
            model=model_name,
            messages=messages,
385
            max_completion_tokens=10,
386
387
            temperature=0.0,
            stream=False,
388
389
            stream_options={"include_usage": None},
        )
390
391
392
393
394
395

    # Test stream=False, stream_options={"include_usage": True}
    with pytest.raises(BadRequestError):
        await client.chat.completions.create(
            model=model_name,
            messages=messages,
396
            max_completion_tokens=10,
397
398
            temperature=0.0,
            stream=False,
399
400
            stream_options={"include_usage": True},
        )
401

402
403
404
405
406
    # Test stream=True, stream_options={"include_usage": True,
    #                           "continuous_usage_stats": True}
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
407
        max_completion_tokens=10,
408
        extra_body=dict(min_tokens=10),
409
410
411
412
        temperature=0.0,
        stream=True,
        stream_options={
            "include_usage": True,
413
            "continuous_usage_stats": True,
414
415
        },
    )
416
    last_completion_tokens = 0
417
418
    async for chunk in stream:
        assert chunk.usage.prompt_tokens >= 0
419
420
421
422
423
424
425
426
427
428
429
        assert (
            last_completion_tokens == 0
            or chunk.usage.completion_tokens > last_completion_tokens
            or (
                not chunk.choices
                and chunk.usage.completion_tokens == last_completion_tokens
            )
        )
        assert chunk.usage.total_tokens == (
            chunk.usage.prompt_tokens + chunk.usage.completion_tokens
        )
430
431
432
        last_completion_tokens = chunk.usage.completion_tokens

    assert last_completion_tokens == 10
433

434
435

@pytest.mark.asyncio
436
async def test_structured_outputs_choice_chat(
437
438
439
    client: openai.AsyncOpenAI,
    sample_structured_outputs_choices,
):
440
441
442
443
444
445
446
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": "The best language for type-safe systems programming is ",
        },
    ]
447
448
449
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
450
        max_completion_tokens=10,
451
        temperature=0.7,
452
        extra_body=dict(
453
454
455
            structured_outputs={"choice": sample_structured_outputs_choices}
        ),
    )
456
    choice1 = chat_completion.choices[0].message.content
457
    assert choice1 in sample_structured_outputs_choices
458
459

    messages.append({"role": "assistant", "content": choice1})
460
    messages.append({"role": "user", "content": "I disagree, pick another one"})
461
462
463
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
464
        max_completion_tokens=10,
465
        temperature=0.7,
466
        extra_body=dict(
467
468
469
            structured_outputs={"choice": sample_structured_outputs_choices}
        ),
    )
470
    choice2 = chat_completion.choices[0].message.content
471
    assert choice2 in sample_structured_outputs_choices
472
473
474
475
    assert choice1 != choice2


@pytest.mark.asyncio
476
477
478
479
async def test_structured_outputs_json_chat(
    client: openai.AsyncOpenAI,
    sample_json_schema,
):
480
481
482
483
484
485
486
487
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": f"Give an example JSON for an employee profile that "
            f"fits this schema: {sample_json_schema}",
        },
    ]
488
489
490
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
491
        max_completion_tokens=1000,
492
493
        extra_body=dict(structured_outputs={"json": sample_json_schema}),
    )
494
495
496
    message = chat_completion.choices[0].message
    assert message.content is not None
    json1 = json.loads(message.content)
497
    jsonschema.validate(instance=json1, schema=sample_json_schema)
498
499

    messages.append({"role": "assistant", "content": message.content})
500
501
502
    messages.append(
        {"role": "user", "content": "Give me another one with a different name and age"}
    )
503
504
505
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
506
        max_completion_tokens=1000,
507
508
        extra_body=dict(structured_outputs={"json": sample_json_schema}),
    )
509
510
511
    message = chat_completion.choices[0].message
    assert message.content is not None
    json2 = json.loads(message.content)
512
    jsonschema.validate(instance=json2, schema=sample_json_schema)
513
514
515
516
517
    assert json1["name"] != json2["name"]
    assert json1["age"] != json2["age"]


@pytest.mark.asyncio
518
519
520
521
async def test_structured_outputs_regex_chat(
    client: openai.AsyncOpenAI,
    sample_regex,
):
522
523
524
525
526
527
528
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": f"Give an example IP address with this regex: {sample_regex}",
        },
    ]
529
530
531
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
532
        max_completion_tokens=20,
533
534
        extra_body=dict(structured_outputs={"regex": sample_regex}),
    )
535
536
    ip1 = chat_completion.choices[0].message.content
    assert ip1 is not None
537
    assert re.fullmatch(sample_regex, ip1) is not None
538
539
540
541
542
543

    messages.append({"role": "assistant", "content": ip1})
    messages.append({"role": "user", "content": "Give me a different one"})
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
544
        max_completion_tokens=20,
545
546
        extra_body=dict(structured_outputs={"regex": sample_regex}),
    )
547
548
    ip2 = chat_completion.choices[0].message.content
    assert ip2 is not None
549
    assert re.fullmatch(sample_regex, ip2) is not None
550
551
552
553
    assert ip1 != ip2


@pytest.mark.asyncio
554
async def test_structured_outputs_type_error(client: openai.AsyncOpenAI):
555
556
557
558
559
560
561
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": "The best language for type-safe systems programming is ",
        },
    ]
562
563

    with pytest.raises(openai.BadRequestError):
564
565
566
        _ = await client.chat.completions.create(
            model=MODEL_NAME,
            messages=messages,
567
568
            extra_body=dict(structured_outputs={"regex": {1: "Python", 2: "C++"}}),
        )
569
570
571


@pytest.mark.asyncio
572
async def test_structured_outputs_choice_chat_logprobs(
573
574
575
576
577
578
579
580
581
    client: openai.AsyncOpenAI, sample_structured_outputs_choices
):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {
            "role": "user",
            "content": "The best language for type-safe systems programming is ",
        },
    ]
582
583
584
    chat_completion = await client.chat.completions.create(
        model=MODEL_NAME,
        messages=messages,
585
        max_completion_tokens=10,
586
587
        logprobs=True,
        top_logprobs=5,
588
        extra_body=dict(
589
590
591
            structured_outputs={"choice": sample_structured_outputs_choices}
        ),
    )
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606

    assert chat_completion.choices[0].logprobs is not None
    assert chat_completion.choices[0].logprobs.content is not None
    top_logprobs = chat_completion.choices[0].logprobs.content[0].top_logprobs

    # -9999.0 is the minimum logprob returned by OpenAI
    for item in top_logprobs:
        assert item.logprob >= -9999.0, f"Failed (top_logprobs={top_logprobs})"


@pytest.mark.asyncio
async def test_response_format_json_object(client: openai.AsyncOpenAI):
    for _ in range(2):
        resp = await client.chat.completions.create(
            model=MODEL_NAME,
607
608
609
610
611
612
613
614
615
616
617
            messages=[
                {
                    "role": "user",
                    "content": (
                        "what is 1+1? please respond with a JSON object, "
                        'the format is {"result": 2}'
                    ),
                }
            ],
            response_format={"type": "json_object"},
        )
618
619
620
621
622
623
624
625

        content = resp.choices[0].message.content
        assert content is not None

        loaded = json.loads(content)
        assert loaded == {"result": 2}, loaded


626
@pytest.mark.asyncio
627
async def test_response_format_json_schema(client: openai.AsyncOpenAI):
628
629
    prompt = 'what is 1+1? The format is "result": 2'
    # Check that this prompt cannot lead to a valid JSON without json_schema
630
631
632
    for _ in range(2):
        resp = await client.chat.completions.create(
            model=MODEL_NAME,
633
            messages=[{"role": "user", "content": prompt}],
634
635
636
637
638
639
640
641
642
643
        )
        content = resp.choices[0].message.content
        assert content is not None
        with pytest.raises((json.JSONDecodeError, AssertionError)):
            loaded = json.loads(content)
            assert loaded == {"result": 2}, loaded

    for _ in range(2):
        resp = await client.chat.completions.create(
            model=MODEL_NAME,
644
            messages=[{"role": "user", "content": prompt}],
645
646
647
648
649
650
651
            response_format={
                "type": "json_schema",
                "json_schema": {
                    "name": "foo_test",
                    "schema": {
                        "type": "object",
                        "properties": {
652
                            "result": {"type": "integer"},
653
654
                        },
                    },
655
656
657
                },
            },
        )
658
659
660
661
662
663
664
665

        content = resp.choices[0].message.content
        assert content is not None

        loaded = json.loads(content)
        assert loaded == {"result": 2}, loaded


666
@pytest.mark.asyncio
667
668
669
async def test_extra_fields_allowed(client: openai.AsyncOpenAI):
    resp = await client.chat.completions.create(
        model=MODEL_NAME,
670
671
672
673
674
675
676
        messages=[
            {
                "role": "user",
                "content": "what is 1+1?",
                "extra_field": "0",
            }
        ],  # type: ignore
677
        temperature=0,
678
679
        seed=0,
    )
680
681
682

    content = resp.choices[0].message.content
    assert content is not None
683
684
685
686


@pytest.mark.asyncio
async def test_complex_message_content(client: openai.AsyncOpenAI):
687
688
689
690
691
692
    content = [
        {
            "type": "text",
            "text": "what is 1+1? please provide the result without any other text.",
        }
    ]
693
694
    resp = await client.chat.completions.create(
        model=MODEL_NAME,
695
696
697
        messages=[
            {
                "role": "user",
698
                "content": content,
699
700
            }
        ],
701
        temperature=0,
702
703
        seed=0,
    )
704
705
706
707
708
709
710
711
712
713
714
    content = resp.choices[0].message.content
    assert content == "2"


@pytest.mark.asyncio
async def test_custom_role(client: openai.AsyncOpenAI):
    # Not sure how the model handles custom roles so we just check that
    # both string and complex message content are handled in the same way

    resp1 = await client.chat.completions.create(
        model=MODEL_NAME,
715
716
717
718
719
720
        messages=[
            {
                "role": "my-custom-role",
                "content": "what is 1+1?",
            }
        ],  # type: ignore
721
        temperature=0,
722
723
        seed=0,
    )
724
725
726

    resp2 = await client.chat.completions.create(
        model=MODEL_NAME,
727
728
729
730
731
732
        messages=[
            {
                "role": "my-custom-role",
                "content": [{"type": "text", "text": "what is 1+1?"}],
            }
        ],  # type: ignore
733
        temperature=0,
734
735
        seed=0,
    )
736
737
738
739
740
741
742
743

    content1 = resp1.choices[0].message.content
    content2 = resp2.choices[0].message.content
    assert content1 == content2


@pytest.mark.asyncio
async def test_long_seed(client: openai.AsyncOpenAI):
744
    for seed in [torch.iinfo(torch.long).min - 1, torch.iinfo(torch.long).max + 1]:
745
746
747
        with pytest.raises(BadRequestError) as exc_info:
            await client.chat.completions.create(
                model=MODEL_NAME,
748
749
750
751
752
753
                messages=[
                    {
                        "role": "system",
                        "content": "You are a helpful assistant.",
                    }
                ],
754
                temperature=0,
755
756
                seed=seed,
            )
757

758
759
760
761
        assert (
            "greater_than_equal" in exc_info.value.message
            or "less_than_equal" in exc_info.value.message
        )
762
763


764
@pytest.mark.asyncio
765
766
767
768
769
async def test_invocations(server: RemoteOpenAIServer, client: openai.AsyncOpenAI):
    messages = [
        {"role": "system", "content": "you are a helpful assistant"},
        {"role": "user", "content": "what is 1+1?"},
    ]
770
771
772
773
774
775
776
777
778
779
780

    request_args = {
        "model": MODEL_NAME,
        "messages": messages,
        "max_completion_tokens": 5,
        "temperature": 0.0,
        "logprobs": False,
    }

    chat_completion = await client.chat.completions.create(**request_args)

781
782
783
    invocation_response = requests.post(
        server.url_for("invocations"), json=request_args
    )
784
785
786
787
788
789
790
    invocation_response.raise_for_status()

    chat_output = chat_completion.model_dump()
    invocation_output = invocation_response.json()

    assert chat_output.keys() == invocation_output.keys()
    assert chat_output["choices"] == invocation_output["choices"]