test_nightly_gsm8k_eval.py 6.39 KB
Newer Older
1
2
import json
import os
3
import unittest
4
import warnings
5
from datetime import datetime
6
7
from types import SimpleNamespace

8
from sglang.srt.utils import kill_process_tree
9
10
11
12
from sglang.test.run_eval import run_eval
from sglang.test.test_utils import (
    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1,
    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2,
13
    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1,
14
15
16
17
    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1,
    DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2,
    DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
    DEFAULT_URL_FOR_TEST,
18
    is_in_ci,
19
    popen_launch_server,
20
    write_github_step_summary,
21
22
)

23
MODEL_SCORE_THRESHOLDS = {
24
    "meta-llama/Llama-3.1-8B-Instruct": 0.82,
25
    "mistralai/Mistral-7B-Instruct-v0.3": 0.58,
26
    "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.85,
27
    "google/gemma-2-27b-it": 0.92,
28
    "meta-llama/Llama-3.1-70B-Instruct": 0.95,
Lianmin Zheng's avatar
Lianmin Zheng committed
29
    "mistralai/Mixtral-8x7B-Instruct-v0.1": 0.64,
30
    "Qwen/Qwen2-57B-A14B-Instruct": 0.86,
31
    "neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8": 0.83,
32
    "neuralmagic/Mistral-7B-Instruct-v0.3-FP8": 0.54,
33
    "neuralmagic/DeepSeek-Coder-V2-Lite-Instruct-FP8": 0.84,
34
    "neuralmagic/gemma-2-2b-it-FP8": 0.60,
35
    "neuralmagic/Meta-Llama-3.1-70B-Instruct-FP8": 0.94,
Lianmin Zheng's avatar
Lianmin Zheng committed
36
    "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8": 0.65,
37
    "neuralmagic/Qwen2-72B-Instruct-FP8": 0.94,
38
39
    "neuralmagic/Qwen2-57B-A14B-Instruct-FP8": 0.82,
    "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4": 0.84,
40
    "hugging-quants/Meta-Llama-3.1-8B-Instruct-GPTQ-INT4": 0.83,
41
    "hugging-quants/Mixtral-8x7B-Instruct-v0.1-AWQ-INT4": 0.62,
42
43
}

44
45
46
47
48

def parse_models(model_string):
    return [model.strip() for model in model_string.split(",") if model.strip()]


49
def popen_launch_server_wrapper(base_url, model, is_fp8, is_tp2):
50
51
52
53
54
55
    other_args = ["--log-level-http", "warning", "--trust-remote-code"]
    if is_fp8:
        if "Llama-3" in model or "gemma-2" in model:
            other_args.extend(["--kv-cache-dtype", "fp8_e5m2"])
        elif "Qwen2-72B-Instruct-FP8" in model:
            other_args.extend(["--quantization", "fp8"])
56
57
        elif "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8" in model:
            other_args.extend([])
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
        else:
            other_args.extend(["--quantization", "fp8", "--kv-cache-dtype", "fp8_e5m2"])
    if is_tp2:
        other_args.extend(["--tp", "2"])
    if "DeepSeek" in model:
        other_args.extend(["--mem-frac", "0.85"])
    if "AWQ" in model:
        other_args.extend(["--quantization", "awq"])
    elif "GPTQ" in model:
        other_args.extend(["--quantization", "gptq"])

    process = popen_launch_server(
        model,
        base_url,
        timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
        other_args=other_args,
    )
    return process


78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
def write_results_to_json(model, metrics, mode="a"):
    result = {
        "timestamp": datetime.now().isoformat(),
        "model": model,
        "metrics": metrics,
        "score": metrics["score"],
    }

    existing_results = []
    if mode == "a" and os.path.exists("results.json"):
        try:
            with open("results.json", "r") as f:
                existing_results = json.load(f)
        except json.JSONDecodeError:
            existing_results = []

    if isinstance(existing_results, list):
        existing_results.append(result)
    else:
        existing_results = [result]

    with open("results.json", "w") as f:
        json.dump(existing_results, f, indent=2)


def check_model_scores(results):
    failed_models = []
105
106
107
    summary = " | model | score | threshold |\n"
    summary += "| ----- | ----- | --------- |\n"

108
109
110
111
112
113
114
115
116
117
118
119
    for model, score in results:
        threshold = MODEL_SCORE_THRESHOLDS.get(model)
        if threshold is None:
            print(f"Warning: No threshold defined for model {model}")
            continue

        if score < threshold:
            failed_models.append(
                f"\nScore Check Failed: {model}\n"
                f"Model {model} score ({score:.4f}) is below threshold ({threshold:.4f})"
            )

120
121
122
123
124
125
126
127
        line = f"| {model} | {score} | {threshold} |\n"
        summary += line

    print(summary)

    if is_in_ci():
        write_github_step_summary(f"### TestNightlyGsm8KEval\n{summary}")

128
129
130
131
    if failed_models:
        raise AssertionError("\n".join(failed_models))


132
class TestNightlyGsm8KEval(unittest.TestCase):
133
134
135
136
137
138
139
    @classmethod
    def setUpClass(cls):
        cls.model_groups = [
            (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP1), False, False),
            (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_TP2), False, True),
            (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP1), True, False),
            (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_FP8_TP2), True, True),
140
            (parse_models(DEFAULT_MODEL_NAME_FOR_NIGHTLY_EVAL_QUANT_TP1), False, False),
141
142
143
144
        ]
        cls.base_url = DEFAULT_URL_FOR_TEST

    def test_mgsm_en_all_models(self):
145
146
147
        warnings.filterwarnings(
            "ignore", category=ResourceWarning, message="unclosed.*socket"
        )
148
149
150
        is_first = True
        all_results = []

151
152
153
        for model_group, is_fp8, is_tp2 in self.model_groups:
            for model in model_group:
                with self.subTest(model=model):
154
155
156
                    process = popen_launch_server_wrapper(
                        self.base_url, model, is_fp8, is_tp2
                    )
157
158
159
160
161
162
163
164
165
166
167
168
169

                    args = SimpleNamespace(
                        base_url=self.base_url,
                        model=model,
                        eval_name="mgsm_en",
                        num_examples=None,
                        num_threads=1024,
                    )

                    metrics = run_eval(args)
                    print(
                        f"{'=' * 42}\n{model} - metrics={metrics} score={metrics['score']}\n{'=' * 42}\n"
                    )
170
171
172
173
174

                    write_results_to_json(model, metrics, "w" if is_first else "a")
                    is_first = False

                    all_results.append((model, metrics["score"]))
175
                    kill_process_tree(process.pid)
176

177
178
179
180
181
182
183
184
185
186
        try:
            with open("results.json", "r") as f:
                print("\nFinal Results from results.json:")
                print(json.dumps(json.load(f), indent=2))
        except Exception as e:
            print(f"Error reading results.json: {e}")

        # Check all scores after collecting all results
        check_model_scores(all_results)

187
188
189

if __name__ == "__main__":
    unittest.main()