test_triton_attention_backend.py 1.48 KB
Newer Older
1
import subprocess
Lianmin Zheng's avatar
Lianmin Zheng committed
2
3
4
5
6
7
8
import unittest
from types import SimpleNamespace

from sglang.srt.utils import kill_child_process
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
    DEFAULT_MODEL_NAME_FOR_TEST,
9
10
    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
    DEFAULT_URL_FOR_TEST,
11
    is_in_ci,
Lianmin Zheng's avatar
Lianmin Zheng committed
12
    popen_launch_server,
13
    run_bench_latency,
Lianmin Zheng's avatar
Lianmin Zheng committed
14
15
16
17
)


class TestTritonAttnBackend(unittest.TestCase):
18
19
20
21
22
23
24
25
    def test_latency(self):
        output_throughput = run_bench_latency(
            DEFAULT_MODEL_NAME_FOR_TEST,
            [
                "--attention-backend",
                "triton",
                "--enable-torch-compile",
            ],
Lianmin Zheng's avatar
Lianmin Zheng committed
26
27
        )

28
        if is_in_ci():
29
            assert output_throughput > 153, f"{output_throughput=}"
Lianmin Zheng's avatar
Lianmin Zheng committed
30
31

    def test_mmlu(self):
32
33
34
35
36
37
38
        model = DEFAULT_MODEL_NAME_FOR_TEST
        base_url = DEFAULT_URL_FOR_TEST
        process = popen_launch_server(
            model,
            base_url,
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
            other_args=["--attention-backend", "triton"],
Lianmin Zheng's avatar
Lianmin Zheng committed
39
40
        )

41
42
43
44
45
46
47
48
49
50
51
52
        try:
            args = SimpleNamespace(
                base_url=base_url,
                model=model,
                eval_name="mmlu",
                num_examples=64,
                num_threads=32,
            )

            metrics = run_eval(args)
            assert metrics["score"] >= 0.65
        finally:
Lianmin Zheng's avatar
Lianmin Zheng committed
53
            kill_child_process(process.pid, include_self=True)
Lianmin Zheng's avatar
Lianmin Zheng committed
54
55
56
57


if __name__ == "__main__":
    unittest.main()