test_vision_openai_server.py 22.7 KB
Newer Older
1
2
3
"""
Usage:
python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_mixed_batch
4
python3 -m unittest test_vision_openai_server.TestOpenAIVisionServer.test_multi_images_chat_completion
5
6
"""

7
8
import base64
import io
Ying Sheng's avatar
Ying Sheng committed
9
import json
10
import os
Ying Sheng's avatar
Ying Sheng committed
11
import unittest
12
from concurrent.futures import ThreadPoolExecutor
Ying Sheng's avatar
Ying Sheng committed
13

14
import numpy as np
Ying Sheng's avatar
Ying Sheng committed
15
import openai
16
17
import requests
from PIL import Image
Ying Sheng's avatar
Ying Sheng committed
18

19
from sglang.srt.utils import kill_process_tree
20
21
22
from sglang.test.test_utils import (
    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
    DEFAULT_URL_FOR_TEST,
23
    CustomTestCase,
24
25
    popen_launch_server,
)
Ying Sheng's avatar
Ying Sheng committed
26

27
28
29
30
31
32
33
34
35
36
37
# image
IMAGE_MAN_IRONING_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/man_ironing_on_back_of_suv.png"
IMAGE_SGL_LOGO_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/images/sgl_logo.png"

# video
VIDEO_JOBS_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/videos/jobs_presenting_ipod.mp4"

# audio
AUDIO_TRUMP_SPEECH_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/Trump_WEF_2018_10s.mp3"
AUDIO_BIRD_SONG_URL = "https://raw.githubusercontent.com/sgl-project/sgl-test-files/refs/heads/main/audios/bird_song.mp3"

Ying Sheng's avatar
Ying Sheng committed
38

39
class TestOpenAIVisionServer(CustomTestCase):
Ying Sheng's avatar
Ying Sheng committed
40
41
    @classmethod
    def setUpClass(cls):
42
        cls.model = "lmms-lab/llava-onevision-qwen2-0.5b-ov"
43
        cls.base_url = DEFAULT_URL_FOR_TEST
Ying Sheng's avatar
Ying Sheng committed
44
45
46
47
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
48
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
Ying Sheng's avatar
Ying Sheng committed
49
50
51
            api_key=cls.api_key,
            other_args=[
                "--chat-template",
52
                "chatml-llava",
53
                # "--log-requests",
Ying Sheng's avatar
Ying Sheng committed
54
55
56
57
58
59
            ],
        )
        cls.base_url += "/v1"

    @classmethod
    def tearDownClass(cls):
60
        kill_process_tree(cls.process.pid)
Ying Sheng's avatar
Ying Sheng committed
61

62
    def test_single_image_chat_completion(self):
Ying Sheng's avatar
Ying Sheng committed
63
64
65
66
67
68
69
70
71
72
        client = openai.Client(api_key=self.api_key, base_url=self.base_url)

        response = client.chat.completions.create(
            model="default",
            messages=[
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "image_url",
73
                            "image_url": {"url": IMAGE_MAN_IRONING_URL},
Ying Sheng's avatar
Ying Sheng committed
74
                        },
Ying Sheng's avatar
Ying Sheng committed
75
76
77
78
                        {
                            "type": "text",
                            "text": "Describe this image in a very short sentence.",
                        },
Ying Sheng's avatar
Ying Sheng committed
79
80
81
82
83
84
85
                    ],
                },
            ],
            temperature=0,
        )

        assert response.choices[0].message.role == "assistant"
Ying Sheng's avatar
Ying Sheng committed
86
87
        text = response.choices[0].message.content
        assert isinstance(text, str)
88
89
        # `driver` is for gemma-3-it
        assert "man" in text or "person" or "driver" in text, text
90
        assert "cab" in text or "taxi" in text or "SUV" in text, text
Mick's avatar
Mick committed
91
92
        # MiniCPMO fails to recognize `iron`, but `hanging`
        assert "iron" in text or "hang" in text, text
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
        assert response.id
        assert response.created
        assert response.usage.prompt_tokens > 0
        assert response.usage.completion_tokens > 0
        assert response.usage.total_tokens > 0

    def test_multi_turn_chat_completion(self):
        client = openai.Client(api_key=self.api_key, base_url=self.base_url)

        response = client.chat.completions.create(
            model="default",
            messages=[
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "image_url",
110
                            "image_url": {"url": IMAGE_MAN_IRONING_URL},
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
                        },
                        {
                            "type": "text",
                            "text": "Describe this image in a very short sentence.",
                        },
                    ],
                },
                {
                    "role": "assistant",
                    "content": [
                        {
                            "type": "text",
                            "text": "There is a man at the back of a yellow cab ironing his clothes.",
                        }
                    ],
                },
                {
                    "role": "user",
                    "content": [
                        {"type": "text", "text": "Repeat your previous answer."}
                    ],
                },
            ],
            temperature=0,
        )

        assert response.choices[0].message.role == "assistant"
        text = response.choices[0].message.content
        assert isinstance(text, str)
140
        assert "man" in text or "cab" in text, text
Ying Sheng's avatar
Ying Sheng committed
141
142
143
144
        assert response.id
        assert response.created
        assert response.usage.prompt_tokens > 0
        assert response.usage.completion_tokens > 0
145
146
        assert response.usage.total_tokens > 0

147
    def test_multi_images_chat_completion(self):
148
149
150
151
152
153
154
155
156
157
        client = openai.Client(api_key=self.api_key, base_url=self.base_url)

        response = client.chat.completions.create(
            model="default",
            messages=[
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "image_url",
Mick's avatar
Mick committed
158
                            "image_url": {"url": IMAGE_MAN_IRONING_URL},
159
                            "modalities": "multi-images",
160
161
162
                        },
                        {
                            "type": "image_url",
163
                            "image_url": {"url": IMAGE_SGL_LOGO_URL},
164
                            "modalities": "multi-images",
165
166
167
                        },
                        {
                            "type": "text",
168
169
                            "text": "I have two very different images. They are not related at all. "
                            "Please describe the first image in one sentence, and then describe the second image in another sentence.",
170
171
172
173
174
175
176
177
178
179
                        },
                    ],
                },
            ],
            temperature=0,
        )

        assert response.choices[0].message.role == "assistant"
        text = response.choices[0].message.content
        assert isinstance(text, str)
Mick's avatar
Mick committed
180
181
182
        print("-" * 30)
        print(f"Multi images response:\n{text}")
        print("-" * 30)
Mick's avatar
Mick committed
183
        assert "man" in text or "cab" in text or "SUV" in text or "taxi" in text, text
184
        assert "logo" in text or '"S"' in text or "SG" in text, text
185
186
187
188
        assert response.id
        assert response.created
        assert response.usage.prompt_tokens > 0
        assert response.usage.completion_tokens > 0
Ying Sheng's avatar
Ying Sheng committed
189
190
        assert response.usage.total_tokens > 0

191
    def prepare_video_messages(self, video_path):
192
193
        # the memory consumed by the Vision Attention varies a lot, e.g. blocked qkv vs full-sequence sdpa
        # the size of the video embeds differs from the `modality` argument when preprocessed
194
195
196
197
198
199
200

        # We import decord here to avoid a strange Segmentation fault (core dumped) issue.
        # The following import order will cause Segmentation fault.
        # import decord
        # from transformers import AutoTokenizer
        from decord import VideoReader, cpu

201
        max_frames_num = 20
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
        vr = VideoReader(video_path, ctx=cpu(0))
        total_frame_num = len(vr)
        uniform_sampled_frames = np.linspace(
            0, total_frame_num - 1, max_frames_num, dtype=int
        )
        frame_idx = uniform_sampled_frames.tolist()
        frames = vr.get_batch(frame_idx).asnumpy()

        base64_frames = []
        for frame in frames:
            pil_img = Image.fromarray(frame)
            buff = io.BytesIO()
            pil_img.save(buff, format="JPEG")
            base64_str = base64.b64encode(buff.getvalue()).decode("utf-8")
            base64_frames.append(base64_str)

        messages = [{"role": "user", "content": []}]
        frame_format = {
            "type": "image_url",
            "image_url": {"url": "data:image/jpeg;base64,{}"},
222
            "modalities": "video",
223
224
225
226
227
228
229
230
231
232
233
234
235
        }

        for base64_frame in base64_frames:
            frame_format["image_url"]["url"] = "data:image/jpeg;base64,{}".format(
                base64_frame
            )
            messages[0]["content"].append(frame_format.copy())

        prompt = {"type": "text", "text": "Please describe the video in detail."}
        messages[0]["content"].append(prompt)

        return messages

236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
    def prepare_video_messages_video_direct(self, video_path):
        messages = [
            {
                "role": "user",
                "content": [
                    {
                        "type": "image_url",
                        "image_url": {"url": f"video:{video_path}"},
                        "modalities": "video",
                    },
                    {"type": "text", "text": "Please describe the video in detail."},
                ],
            },
        ]
        return messages

252
    def get_or_download_file(self, url: str) -> str:
253
        cache_dir = os.path.expanduser("~/.cache")
254
255
256
257
        if url is None:
            raise ValueError()
        file_name = url.split("/")[-1]
        file_path = os.path.join(cache_dir, file_name)
258
259
260
261
262
263
264
265
        os.makedirs(cache_dir, exist_ok=True)

        if not os.path.exists(file_path):
            response = requests.get(url)
            response.raise_for_status()

            with open(file_path, "wb") as f:
                f.write(response.content)
266
267
268
269
270
        return file_path

    def test_video_chat_completion(self):
        url = VIDEO_JOBS_URL
        file_path = self.get_or_download_file(url)
271
272
273

        client = openai.Client(api_key=self.api_key, base_url=self.base_url)

274
        # messages = self.prepare_video_messages_video_direct(file_path)
275
276
        messages = self.prepare_video_messages(file_path)

Mick's avatar
Mick committed
277
        response = client.chat.completions.create(
278
279
280
281
            model="default",
            messages=messages,
            temperature=0,
            max_tokens=1024,
Mick's avatar
Mick committed
282
            stream=False,
283
        )
284

Mick's avatar
Mick committed
285
286
        video_response = response.choices[0].message.content

287
        print("-" * 30)
Mick's avatar
Mick committed
288
        print(f"Video response:\n{video_response}")
289
290
291
        print("-" * 30)

        # Add assertions to validate the video response
Mick's avatar
Mick committed
292
293
294
295
296
        assert "iPod" in video_response or "device" in video_response, video_response
        assert (
            "man" in video_response
            or "person" in video_response
            or "individual" in video_response
297
            or "speaker" in video_response
Mick's avatar
Mick committed
298
299
300
301
302
        ), video_response
        assert (
            "present" in video_response
            or "examine" in video_response
            or "display" in video_response
303
            or "hold" in video_response
Mick's avatar
Mick committed
304
305
        )
        assert "black" in video_response or "dark" in video_response
306
307
308
        self.assertIsNotNone(video_response)
        self.assertGreater(len(video_response), 0)

Ying Sheng's avatar
Ying Sheng committed
309
    def test_regex(self):
Mick's avatar
Mick committed
310
        return
Ying Sheng's avatar
Ying Sheng committed
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
        client = openai.Client(api_key=self.api_key, base_url=self.base_url)

        regex = (
            r"""\{\n"""
            + r"""   "color": "[\w]+",\n"""
            + r"""   "number_of_cars": [\d]+\n"""
            + r"""\}"""
        )

        response = client.chat.completions.create(
            model="default",
            messages=[
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "image_url",
328
                            "image_url": {"url": IMAGE_MAN_IRONING_URL},
Ying Sheng's avatar
Ying Sheng committed
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
                        },
                        {
                            "type": "text",
                            "text": "Describe this image in the JSON format.",
                        },
                    ],
                },
            ],
            temperature=0,
            extra_body={"regex": regex},
        )
        text = response.choices[0].message.content

        try:
            js_obj = json.loads(text)
        except (TypeError, json.decoder.JSONDecodeError):
            print("JSONDecodeError", text)
            raise
        assert isinstance(js_obj["color"], str)
        assert isinstance(js_obj["number_of_cars"], int)

350
351
352
353
354
355
356
357
    def run_decode_with_image(self, image_id):
        client = openai.Client(api_key=self.api_key, base_url=self.base_url)

        content = []
        if image_id == 0:
            content.append(
                {
                    "type": "image_url",
358
                    "image_url": {"url": IMAGE_MAN_IRONING_URL},
359
360
361
362
363
364
                }
            )
        elif image_id == 1:
            content.append(
                {
                    "type": "image_url",
365
                    "image_url": {"url": IMAGE_SGL_LOGO_URL},
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
                }
            )
        else:
            pass

        content.append(
            {
                "type": "text",
                "text": "Describe this image in a very short sentence.",
            }
        )

        response = client.chat.completions.create(
            model="default",
            messages=[
                {"role": "user", "content": content},
            ],
            temperature=0,
        )

        assert response.choices[0].message.role == "assistant"
        text = response.choices[0].message.content
        assert isinstance(text, str)

    def test_mixed_batch(self):
        image_ids = [0, 1, 2] * 4
        with ThreadPoolExecutor(4) as executor:
            list(executor.map(self.run_decode_with_image, image_ids))

Mick's avatar
Mick committed
395
396
397
398
399
400
401
402
403
    def prepare_audio_messages(self, prompt, audio_file_name):
        messages = [
            {
                "role": "user",
                "content": [
                    {
                        "type": "audio_url",
                        "audio_url": {"url": f"{audio_file_name}"},
                    },
Mick's avatar
Mick committed
404
405
406
407
                    {
                        "type": "text",
                        "text": prompt,
                    },
Mick's avatar
Mick committed
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
                ],
            }
        ]

        return messages

    def get_audio_response(self, url: str, prompt, category):
        audio_file_path = self.get_or_download_file(url)
        client = openai.Client(api_key="sk-123456", base_url=self.base_url)

        messages = self.prepare_audio_messages(prompt, audio_file_path)

        response = client.chat.completions.create(
            model="default",
            messages=messages,
            temperature=0,
            max_tokens=128,
            stream=False,
        )

        audio_response = response.choices[0].message.content

        print("-" * 30)
        print(f"audio {category} response:\n{audio_response}")
        print("-" * 30)

        audio_response = audio_response.lower()

        self.assertIsNotNone(audio_response)
        self.assertGreater(len(audio_response), 0)

        return audio_response

    def _test_audio_speech_completion(self):
        # a fragment of Trump's speech
        audio_response = self.get_audio_response(
            AUDIO_TRUMP_SPEECH_URL,
            "I have an audio sample. Please repeat the person's words",
            category="speech",
        )
        assert "thank you" in audio_response
        assert "it's a privilege to be here" in audio_response
        assert "leader" in audio_response
        assert "science" in audio_response
        assert "art" in audio_response

    def _test_audio_ambient_completion(self):
        # bird song
        audio_response = self.get_audio_response(
            AUDIO_BIRD_SONG_URL,
            "Please listen to the audio snippet carefully and transcribe the content.",
            "ambient",
        )
        assert "bird" in audio_response

    def test_audio_chat_completion(self):
        pass

Ying Sheng's avatar
Ying Sheng committed
466

467
class TestQwen2VLServer(TestOpenAIVisionServer):
Yineng Zhang's avatar
Yineng Zhang committed
468
469
470
471
472
473
474
475
476
477
478
479
480
    @classmethod
    def setUpClass(cls):
        cls.model = "Qwen/Qwen2-VL-7B-Instruct"
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            api_key=cls.api_key,
            other_args=[
                "--chat-template",
                "qwen2-vl",
481
482
                "--mem-fraction-static",
                "0.4",
Yineng Zhang's avatar
Yineng Zhang committed
483
484
485
486
487
            ],
        )
        cls.base_url += "/v1"


488
class TestQwen2_5_VLServer(TestOpenAIVisionServer):
Mick's avatar
Mick committed
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
    @classmethod
    def setUpClass(cls):
        cls.model = "Qwen/Qwen2.5-VL-7B-Instruct"
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            api_key=cls.api_key,
            other_args=[
                "--chat-template",
                "qwen2-vl",
                "--mem-fraction-static",
                "0.4",
            ],
        )
        cls.base_url += "/v1"


509
class TestVLMContextLengthIssue(CustomTestCase):
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
    @classmethod
    def setUpClass(cls):
        cls.model = "Qwen/Qwen2-VL-7B-Instruct"
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            api_key=cls.api_key,
            other_args=[
                "--chat-template",
                "qwen2-vl",
                "--context-length",
                "300",
                "--mem-fraction-static=0.80",
            ],
        )
        cls.base_url += "/v1"

    @classmethod
    def tearDownClass(cls):
532
        kill_process_tree(cls.process.pid)
533

534
    def test_single_image_chat_completion(self):
535
536
        client = openai.Client(api_key=self.api_key, base_url=self.base_url)

537
538
539
540
541
542
543
544
545
        with self.assertRaises(openai.BadRequestError) as cm:
            client.chat.completions.create(
                model="default",
                messages=[
                    {
                        "role": "user",
                        "content": [
                            {
                                "type": "image_url",
546
                                "image_url": {"url": IMAGE_MAN_IRONING_URL},
547
                            },
548
549
550
551
552
553
554
555
556
                            {
                                "type": "text",
                                "text": "Give a lengthy description of this picture",
                            },
                        ],
                    },
                ],
                temperature=0,
            )
557

558
559
560
561
562
        # context length is checked first, then max_req_input_len, which is calculated from the former
        assert (
            "Multimodal prompt is too long after expanding multimodal tokens."
            in str(cm.exception)
            or "is longer than the model's context length" in str(cm.exception)
563
        )
564
565


566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
class TestMllamaServer(TestOpenAIVisionServer):
    @classmethod
    def setUpClass(cls):
        cls.model = "meta-llama/Llama-3.2-11B-Vision-Instruct"
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            api_key=cls.api_key,
            other_args=[
                "--chat-template",
                "llama_3_vision",
            ],
        )
        cls.base_url += "/v1"

    def test_video_chat_completion(self):
        pass


Mick's avatar
Mick committed
588
589
590
591
592
593
594
595
596
597
598
599
600
601
class TestMinicpmvServer(TestOpenAIVisionServer):
    @classmethod
    def setUpClass(cls):
        cls.model = "openbmb/MiniCPM-V-2_6"
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            other_args=[
                "--trust-remote-code",
                "--chat-template",
                "minicpmv",
602
603
                "--mem-fraction-static",
                "0.4",
Mick's avatar
Mick committed
604
605
606
607
608
            ],
        )
        cls.base_url += "/v1"


Mick's avatar
Mick committed
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
class TestMinicpmoServer(TestOpenAIVisionServer):
    @classmethod
    def setUpClass(cls):
        cls.model = "openbmb/MiniCPM-o-2_6"
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            other_args=[
                "--trust-remote-code",
                "--chat-template",
                "minicpmo",
                "--mem-fraction-static",
                "0.7",
            ],
        )
        cls.base_url += "/v1"

    def test_audio_chat_completion(self):
        self._test_audio_speech_completion()
        self._test_audio_ambient_completion()


634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
class TestDeepseekVL2Server(TestOpenAIVisionServer):
    @classmethod
    def setUpClass(cls):
        cls.model = "deepseek-ai/deepseek-vl2-small"
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            other_args=[
                "--trust-remote-code",
                "--chat-template",
                "deepseek-vl2",
                "--context-length",
                "4096",
            ],
        )
        cls.base_url += "/v1"

    def test_video_chat_completion(self):
        pass


Mick's avatar
Mick committed
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
class TestJanusProServer(TestOpenAIVisionServer):
    @classmethod
    def setUpClass(cls):
        cls.model = "deepseek-ai/Janus-Pro-7B"
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            other_args=[
                "--trust-remote-code",
                "--chat-template",
                "janus-pro",
                "--mem-fraction-static",
                "0.4",
            ],
        )
        cls.base_url += "/v1"

    def test_video_chat_completion(self):
        pass

681
682
683
684
    def test_single_image_chat_completion(self):
        # Skip this test because it is flaky
        pass

Mick's avatar
Mick committed
685

686
687
688
689
690
691
692
693
694
695
696
697
698
699
class TestGemma3itServer(TestOpenAIVisionServer):
    @classmethod
    def setUpClass(cls):
        cls.model = "google/gemma-3-4b-it"
        cls.base_url = DEFAULT_URL_FOR_TEST
        cls.api_key = "sk-123456"
        cls.process = popen_launch_server(
            cls.model,
            cls.base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            other_args=[
                "--trust-remote-code",
                "--chat-template",
                "gemma-it",
700
701
                "--mem-fraction-static",
                "0.75",
702
703
704
705
706
707
708
709
            ],
        )
        cls.base_url += "/v1"

    def test_video_chat_completion(self):
        pass


Ying Sheng's avatar
Ying Sheng committed
710
if __name__ == "__main__":
Lianmin Zheng's avatar
Lianmin Zheng committed
711
    unittest.main()