test_metrics.py 3.05 KB
Newer Older
Yudi Xue's avatar
Yudi Xue committed
1
2
3
4
import unittest

import requests

5
from sglang.srt.utils import kill_process_tree
Yudi Xue's avatar
Yudi Xue committed
6
from sglang.test.test_utils import (
Lianmin Zheng's avatar
Lianmin Zheng committed
7
    DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
Yudi Xue's avatar
Yudi Xue committed
8
9
    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
    DEFAULT_URL_FOR_TEST,
10
    CustomTestCase,
Yudi Xue's avatar
Yudi Xue committed
11
12
13
14
    popen_launch_server,
)


15
class TestEnableMetrics(CustomTestCase):
Yudi Xue's avatar
Yudi Xue committed
16
17
18
    def test_metrics_enabled(self):
        """Test that metrics endpoint returns data when enabled"""
        process = popen_launch_server(
Lianmin Zheng's avatar
Lianmin Zheng committed
19
20
            DEFAULT_SMALL_MODEL_NAME_FOR_TEST,
            DEFAULT_URL_FOR_TEST,
Yudi Xue's avatar
Yudi Xue committed
21
            timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
22
            other_args=["--enable-metrics", "--cuda-graph-max-bs", 2],
Yudi Xue's avatar
Yudi Xue committed
23
24
25
        )

        try:
26
            # Make some requests to generate some metrics
Yudi Xue's avatar
Yudi Xue committed
27
28
29
            response = requests.get(f"{DEFAULT_URL_FOR_TEST}/health_generate")
            self.assertEqual(response.status_code, 200)

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
            response = requests.post(
                f"{DEFAULT_URL_FOR_TEST}/generate",
                json={
                    "text": "The capital of France is",
                    "sampling_params": {
                        "temperature": 0,
                        "max_new_tokens": 32,
                    },
                    "stream": True,
                },
                stream=True,
            )
            for _ in response.iter_lines(decode_unicode=False):
                pass

Yudi Xue's avatar
Yudi Xue committed
45
46
47
48
49
            # Get metrics
            metrics_response = requests.get(f"{DEFAULT_URL_FOR_TEST}/metrics")
            self.assertEqual(metrics_response.status_code, 200)
            metrics_content = metrics_response.text

50
            print(f"metrics_content=\n{metrics_content}")
Lianmin Zheng's avatar
Lianmin Zheng committed
51

Yudi Xue's avatar
Yudi Xue committed
52
53
            # Verify essential metrics are present
            essential_metrics = [
54
                "sglang:num_running_reqs",
55
                "sglang:num_used_tokens",
56
57
                "sglang:token_usage",
                "sglang:gen_throughput",
58
                "sglang:num_queue_reqs",
59
                "sglang:num_grammar_queue_reqs",
60
                "sglang:cache_hit_rate",
61
                "sglang:spec_accept_length",
Yudi Xue's avatar
Yudi Xue committed
62
63
                "sglang:prompt_tokens_total",
                "sglang:generation_tokens_total",
64
                "sglang:cached_tokens_total",
65
                "sglang:num_requests_total",
Yudi Xue's avatar
Yudi Xue committed
66
                "sglang:time_to_first_token_seconds",
67
                "sglang:inter_token_latency_seconds",
Yudi Xue's avatar
Yudi Xue committed
68
69
70
71
72
73
74
                "sglang:e2e_request_latency_seconds",
            ]

            for metric in essential_metrics:
                self.assertIn(metric, metrics_content, f"Missing metric: {metric}")

            # Verify model name label is present and correct
Lianmin Zheng's avatar
Lianmin Zheng committed
75
            expected_model_name = DEFAULT_SMALL_MODEL_NAME_FOR_TEST
Yudi Xue's avatar
Yudi Xue committed
76
            self.assertIn(f'model_name="{expected_model_name}"', metrics_content)
77

Yudi Xue's avatar
Yudi Xue committed
78
79
80
81
82
83
            # Verify metrics have values (not empty)
            self.assertIn("_sum{", metrics_content)
            self.assertIn("_count{", metrics_content)
            self.assertIn("_bucket{", metrics_content)

        finally:
84
            kill_process_tree(process.pid)
Yudi Xue's avatar
Yudi Xue committed
85
86


Lianmin Zheng's avatar
Lianmin Zheng committed
87
88
if __name__ == "__main__":
    unittest.main()