test_bench_serving.py 3.87 KB
Newer Older
1
import sys
2
3
4
import unittest

from sglang.test.test_utils import (
5
    DEFAULT_FP8_MODEL_NAME_FOR_TEST,
6
7
    DEFAULT_MODEL_NAME_FOR_TEST,
    DEFAULT_MOE_MODEL_NAME_FOR_TEST,
8
    is_in_ci,
9
10
11
12
13
14
15
16
17
18
19
20
21
22
    run_bench_serving,
)


class TestBenchServing(unittest.TestCase):

    def test_offline_throughput_default(self):
        res = run_bench_serving(
            model=DEFAULT_MODEL_NAME_FOR_TEST,
            num_prompts=500,
            request_rate=float("inf"),
            other_server_args=[],
        )

23
        if is_in_ci():
24
25
26
27
28
29
30
            assert res["output_throughput"] > 2830

    def test_offline_throughput_non_stream_small_batch_size(self):
        res = run_bench_serving(
            model=DEFAULT_MODEL_NAME_FOR_TEST,
            num_prompts=200,
            request_rate=float("inf"),
31
            other_server_args=["--max-running-requests", "10"],
32
33
34
35
36
37
38
            dataset_name="sharegpt",
            random_input_len=None,
            random_output_len=None,
            disable_stream=True,
        )

        if is_in_ci():
39
40
41
42
43
44
            print(
                f"Output throughput: {res['output_throughput']}, Is greater than 1000: {res['output_throughput'] > 1000}",
                file=sys.stderr,
            )
            # TODO(zhyncs) fix this
            # assert res["output_throughput"] > 1000
45
46
47
48
49
50
51
52
53

    def test_offline_throughput_without_radix_cache(self):
        res = run_bench_serving(
            model=DEFAULT_MODEL_NAME_FOR_TEST,
            num_prompts=500,
            request_rate=float("inf"),
            other_server_args=["--disable-radix-cache"],
        )

54
        if is_in_ci():
55
            assert res["output_throughput"] > 2880
56
57
58
59
60
61
62
63
64

    def test_offline_throughput_without_chunked_prefill(self):
        res = run_bench_serving(
            model=DEFAULT_MODEL_NAME_FOR_TEST,
            num_prompts=500,
            request_rate=float("inf"),
            other_server_args=["--chunked-prefill-size", "-1"],
        )

65
        if is_in_ci():
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
            assert res["output_throughput"] > 2600

    def test_offline_throughput_with_triton_attention_backend(self):
        res = run_bench_serving(
            model=DEFAULT_MODEL_NAME_FOR_TEST,
            num_prompts=500,
            request_rate=float("inf"),
            other_server_args=[
                "--attention-backend",
                "triton",
                "--context-length",
                "8192",
            ],
        )

81
        if is_in_ci():
82
            assert res["output_throughput"] > 2930
83

84
85
86
87
88
89
90
91
92
93
94
    def test_offline_throughput_default_fp8(self):
        res = run_bench_serving(
            model=DEFAULT_FP8_MODEL_NAME_FOR_TEST,
            num_prompts=500,
            request_rate=float("inf"),
            other_server_args=[],
        )

        if is_in_ci():
            assert res["output_throughput"] > 3100

95
96
97
98
99
100
101
102
    def test_online_latency_default(self):
        res = run_bench_serving(
            model=DEFAULT_MODEL_NAME_FOR_TEST,
            num_prompts=100,
            request_rate=1,
            other_server_args=[],
        )

103
        if is_in_ci():
104
            assert res["median_e2e_latency_ms"] < 12000
Lianmin Zheng's avatar
Lianmin Zheng committed
105
            assert res["median_ttft_ms"] < 80
106
107
108
109
110
111
112
113
114
115
            assert res["median_itl_ms"] < 12

    def test_moe_offline_throughput_default(self):
        res = run_bench_serving(
            model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
            num_prompts=300,
            request_rate=float("inf"),
            other_server_args=["--tp", "2"],
        )

116
        if is_in_ci():
117
118
119
120
121
122
123
124
125
126
            assert res["output_throughput"] > 1850

    def test_moe_offline_throughput_without_radix_cache(self):
        res = run_bench_serving(
            model=DEFAULT_MOE_MODEL_NAME_FOR_TEST,
            num_prompts=300,
            request_rate=float("inf"),
            other_server_args=["--tp", "2", "--disable-radix-cache"],
        )

127
        if is_in_ci():
128
129
130
131
132
            assert res["output_throughput"] > 1950


if __name__ == "__main__":
    unittest.main()