test_audio.py 12.1 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
5
import json

6
7
import openai
import pytest
8
import pytest_asyncio
9

10
from tests.utils import RemoteOpenAIServer
11
from vllm.assets.audio import AudioAsset
12
from vllm.multimodal.utils import encode_audio_base64, encode_audio_url, fetch_audio
13

14
MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
15
TEST_AUDIO_URLS = [
16
    AudioAsset("winning_call").url,
17
    AudioAsset("mary_had_lamb").url,
18
]
19
MAXIMUM_AUDIOS = 2
20
21


22
23
24
@pytest.fixture(scope="module")
def server():
    args = [
25
26
        "--dtype",
        "float32",
27
        "--max-model-len",
28
29
30
        "2048",
        "--max-num-seqs",
        "5",
31
        "--enforce-eager",
32
        "--trust-remote-code",
33
        "--limit-mm-per-prompt",
34
        json.dumps({"audio": MAXIMUM_AUDIOS}),
35
36
37
38
    ]

    with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
        yield remote_server
39
40


41
42
43
44
@pytest_asyncio.fixture
async def client(server):
    async with server.get_async_client() as async_client:
        yield async_client
45
46
47


@pytest.fixture(scope="session")
48
def base64_encoded_audio() -> dict[str, str]:
49
50
51
52
53
54
    return {
        audio_url: encode_audio_base64(*fetch_audio(audio_url))
        for audio_url in TEST_AUDIO_URLS
    }


55
56
57
58
59
60
61
62
@pytest.fixture(scope="session")
def url_encoded_audio() -> dict[str, str]:
    return {
        audio_url: encode_audio_url(*fetch_audio(audio_url))
        for audio_url in TEST_AUDIO_URLS
    }


63
64
65
def dummy_messages_from_audio_url(
    audio_urls: str | list[str],
    content_text: str = "What's happening in this audio?",
66
):
67
68
69
70
    if isinstance(audio_urls, str):
        audio_urls = [audio_urls]

    return [
71
72
73
        {
            "role": "user",
            "content": [
74
75
76
77
78
                *(
                    {"type": "audio_url", "audio_url": {"url": audio_url}}
                    for audio_url in audio_urls
                ),
                {"type": "text", "text": content_text},
79
80
81
            ],
        }
    ]
82

83
84
85
86
87
88
89
90
91

@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("audio_url", [TEST_AUDIO_URLS[0]])
async def test_single_chat_session_audio(
    client: openai.AsyncOpenAI, model_name: str, audio_url: str
):
    messages = dummy_messages_from_audio_url(audio_url)

92
    # test single completion
93
94
95
96
97
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
        logprobs=True,
98
        temperature=0.0,
99
100
        top_logprobs=5,
    )
101
102
103
104
105
    assert len(chat_completion.choices) == 1

    choice = chat_completion.choices[0]
    assert choice.finish_reason == "length"
    assert chat_completion.usage == openai.types.CompletionUsage(
106
107
        completion_tokens=10, prompt_tokens=202, total_tokens=212
    )
108
109
110
111
112
113
114
115
116
117
118
119

    message = choice.message
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 10
    assert message.role == "assistant"
    messages.append({"role": "assistant", "content": message.content})

    # test multi-turn dialogue
    messages.append({"role": "user", "content": "express your result in json"})
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
120
        max_completion_tokens=10,
121
122
123
124
125
    )
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 0


126
127
128
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("audio_url", [TEST_AUDIO_URLS[0]])
129
130
131
132
133
134
135
136
137
138
139
140
async def test_error_on_invalid_audio_url_type(
    client: openai.AsyncOpenAI, model_name: str, audio_url: str
):
    messages = [
        {
            "role": "user",
            "content": [
                {"type": "audio_url", "audio_url": audio_url},
                {"type": "text", "text": "What's happening in this audio?"},
            ],
        }
    ]
141
142
143

    # audio_url should be a dict {"url": "some url"}, not directly a string
    with pytest.raises(openai.BadRequestError):
144
145
146
147
148
149
        _ = await client.chat.completions.create(
            model=model_name,
            messages=messages,
            max_completion_tokens=10,
            temperature=0.0,
        )
150
151


152
153
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
154
@pytest.mark.parametrize("audio_url", [TEST_AUDIO_URLS[0]])
155
async def test_single_chat_session_audio_base64encoded(
156
157
158
    client: openai.AsyncOpenAI,
    model_name: str,
    audio_url: str,
159
    url_encoded_audio: dict[str, str],
160
):
161
    messages = dummy_messages_from_audio_url(url_encoded_audio[audio_url])
162
163

    # test single completion
164
165
166
167
168
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
        logprobs=True,
169
        temperature=0.0,
170
171
        top_logprobs=5,
    )
172
173
174
175
176
    assert len(chat_completion.choices) == 1

    choice = chat_completion.choices[0]
    assert choice.finish_reason == "length"
    assert chat_completion.usage == openai.types.CompletionUsage(
177
178
        completion_tokens=10, prompt_tokens=202, total_tokens=212
    )
179
180
181
182
183
184
185
186
187
188
189
190

    message = choice.message
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 10
    assert message.role == "assistant"
    messages.append({"role": "assistant", "content": message.content})

    # test multi-turn dialogue
    messages.append({"role": "user", "content": "express your result in json"})
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
191
        max_completion_tokens=10,
192
        temperature=0.0,
193
194
195
196
197
    )
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 0


198
199
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
200
@pytest.mark.parametrize("audio_url", [TEST_AUDIO_URLS[0]])
201
async def test_single_chat_session_input_audio(
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
    client: openai.AsyncOpenAI,
    model_name: str,
    audio_url: str,
    base64_encoded_audio: dict[str, str],
):
    messages = [
        {
            "role": "user",
            "content": [
                {
                    "type": "input_audio",
                    "input_audio": {
                        "data": base64_encoded_audio[audio_url],
                        "format": "wav",
                    },
                },
                {"type": "text", "text": "What's happening in this audio?"},
            ],
        }
    ]
222
223
224
225
226
227
228

    # test single completion
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
        logprobs=True,
229
230
        top_logprobs=5,
    )
231
232
233
234
235
    assert len(chat_completion.choices) == 1

    choice = chat_completion.choices[0]
    assert choice.finish_reason == "length"
    assert chat_completion.usage == openai.types.CompletionUsage(
236
237
        completion_tokens=10, prompt_tokens=202, total_tokens=212
    )
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255

    message = choice.message
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 10
    assert message.role == "assistant"
    messages.append({"role": "assistant", "content": message.content})

    # test multi-turn dialogue
    messages.append({"role": "user", "content": "express your result in json"})
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
        max_completion_tokens=10,
    )
    message = chat_completion.choices[0].message
    assert message.content is not None and len(message.content) >= 0


256
257
258
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
259
260
261
async def test_chat_streaming_audio(
    client: openai.AsyncOpenAI, model_name: str, audio_url: str
):
262
263
264
    messages = dummy_messages_from_audio_url(
        audio_url, "What's a short title for this audio?"
    )
265
266
267
268
269

    # test single completion
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
270
        max_completion_tokens=8,
271
272
273
274
275
276
277
278
279
        temperature=0.0,
    )
    output = chat_completion.choices[0].message.content
    stop_reason = chat_completion.choices[0].finish_reason

    # test streaming
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
280
        max_completion_tokens=8,
281
282
283
        temperature=0.0,
        stream=True,
    )
284
    chunks: list[str] = []
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
    finish_reason_count = 0
    async for chunk in stream:
        delta = chunk.choices[0].delta
        if delta.role:
            assert delta.role == "assistant"
        if delta.content:
            chunks.append(delta.content)
        if chunk.choices[0].finish_reason is not None:
            finish_reason_count += 1
    # finish reason should only return in last block
    assert finish_reason_count == 1
    assert chunk.choices[0].finish_reason == stop_reason
    assert delta.content
    assert "".join(chunks) == output


301
302
303
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
@pytest.mark.parametrize("audio_url", TEST_AUDIO_URLS)
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
async def test_chat_streaming_input_audio(
    client: openai.AsyncOpenAI,
    model_name: str,
    audio_url: str,
    base64_encoded_audio: dict[str, str],
):
    messages = [
        {
            "role": "user",
            "content": [
                {
                    "type": "input_audio",
                    "input_audio": {
                        "data": base64_encoded_audio[audio_url],
                        "format": "wav",
                    },
                },
321
                {"type": "text", "text": "What's a short title for this audio?"},
322
323
324
            ],
        }
    ]
325
326
327
328
329

    # test single completion
    chat_completion = await client.chat.completions.create(
        model=model_name,
        messages=messages,
330
        max_completion_tokens=8,
331
332
333
334
335
336
337
338
339
        temperature=0.0,
    )
    output = chat_completion.choices[0].message.content
    stop_reason = chat_completion.choices[0].finish_reason

    # test streaming
    stream = await client.chat.completions.create(
        model=model_name,
        messages=messages,
340
        max_completion_tokens=8,
341
342
343
        temperature=0.0,
        stream=True,
    )
344
    chunks: list[str] = []
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
    finish_reason_count = 0
    async for chunk in stream:
        delta = chunk.choices[0].delta
        if delta.role:
            assert delta.role == "assistant"
        if delta.content:
            chunks.append(delta.content)
        if chunk.choices[0].finish_reason is not None:
            finish_reason_count += 1
    # finish reason should only return in last block
    assert finish_reason_count == 1
    assert chunk.choices[0].finish_reason == stop_reason
    assert delta.content
    assert "".join(chunks) == output


361
362
@pytest.mark.asyncio
@pytest.mark.parametrize("model_name", [MODEL_NAME])
363
@pytest.mark.parametrize(
364
365
366
367
368
    "audio_urls", [TEST_AUDIO_URLS, TEST_AUDIO_URLS + [TEST_AUDIO_URLS[0]]]
)
async def test_multi_audio_input(
    client: openai.AsyncOpenAI, model_name: str, audio_urls: list[str]
):
369
    messages = dummy_messages_from_audio_url(audio_urls)
370

371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
    if len(audio_urls) > MAXIMUM_AUDIOS:
        with pytest.raises(openai.BadRequestError):  # test multi-audio input
            await client.chat.completions.create(
                model=model_name,
                messages=messages,
                max_completion_tokens=10,
                temperature=0.0,
            )

        # the server should still work afterwards
        completion = await client.completions.create(
            model=model_name,
            prompt=[0, 0, 0, 0, 0],
            max_tokens=5,
            temperature=0.0,
        )
        completion = completion.choices[0].text
        assert completion is not None and len(completion) >= 0
    else:
        chat_completion = await client.chat.completions.create(
391
392
            model=model_name,
            messages=messages,
393
            max_completion_tokens=10,
394
395
            temperature=0.0,
        )
396
397
        message = chat_completion.choices[0].message
        assert message.content is not None and len(message.content) >= 0