run_suite.py 29.1 KB
Newer Older
1
2
import argparse
import glob
Lianmin Zheng's avatar
Lianmin Zheng committed
3
from dataclasses import dataclass
4
from pathlib import Path
5
6
7

from sglang.test.test_utils import run_unittest_files

Lianmin Zheng's avatar
Lianmin Zheng committed
8
9
10
11
12
13
14

@dataclass
class TestFile:
    name: str
    estimated_time: float = 60


15
# NOTE: please sort the test cases alphabetically by the test file name
16
suites = {
Lianmin Zheng's avatar
Lianmin Zheng committed
17
    "per-commit-1-gpu": [
18
        TestFile("debug_utils/test_tensor_dump_forward_hook.py", 15),
19
        TestFile("function_call/test_json_schema_constraint.py", 1),
20
        TestFile("hicache/test_hicache.py", 116),
21
        TestFile("hicache/test_hicache_eagle.py", 150),
22
23
        TestFile("hicache/test_hicache_mla.py", 127),
        TestFile("hicache/test_hicache_storage.py", 127),
24
25
        TestFile("layers/attention/mamba/test_causal_conv1d.py", 25),
        TestFile("layers/attention/mamba/test_mamba_ssm.py", 50),
26
27
28
29
        TestFile("layers/attention/mamba/test_mamba_ssm_ssd.py", 20),
        TestFile("lora/test_lora.py", 150),
        TestFile("lora/test_lora_eviction.py", 240),
        TestFile("lora/test_lora_update.py", 600),
30
        TestFile("lora/test_lora_backend.py", 99),
31
32
33
        TestFile("lora/test_multi_lora_backend.py", 60),
        TestFile("models/test_compressed_tensors_models.py", 42),
        TestFile("models/test_cross_encoder_models.py", 100),
34
        TestFile("models/test_embedding_models.py", 73),
35
        TestFile("models/test_encoder_embedding_models.py", 460),
Lianmin Zheng's avatar
Lianmin Zheng committed
36
        TestFile("models/test_generation_models.py", 103),
37
        TestFile("models/test_nvidia_nemotron_nano_v2.py", 160),
Lianmin Zheng's avatar
Lianmin Zheng committed
38
        TestFile("models/test_qwen_models.py", 82),
Lianmin Zheng's avatar
Lianmin Zheng committed
39
        TestFile("models/test_reward_models.py", 132),
40
        TestFile("models/test_transformers_models.py", 320),
41
        TestFile("models/test_vlm_models.py", 741),
42
43
        TestFile("openai_server/basic/test_openai_embedding.py", 79),
        TestFile("openai_server/basic/test_openai_server.py", 270),
44
45
46
47
48
        TestFile("openai_server/basic/test_protocol.py", 10),
        TestFile("openai_server/basic/test_serving_chat.py", 10),
        TestFile("openai_server/basic/test_serving_completions.py", 10),
        TestFile("openai_server/basic/test_serving_embedding.py", 10),
        TestFile("openai_server/features/test_enable_thinking.py", 70),
49
50
        TestFile("openai_server/features/test_json_mode.py", 120),
        TestFile("openai_server/features/test_openai_server_ebnf.py", 20),
51
52
53
        TestFile("openai_server/features/test_openai_server_hidden_states.py", 240),
        TestFile("openai_server/features/test_reasoning_content.py", 89),
        TestFile("openai_server/function_call/test_openai_function_calling.py", 60),
54
        TestFile("openai_server/function_call/test_tool_choice.py", 120),
55
56
57
58
        TestFile("openai_server/validation/test_large_max_new_tokens.py", 41),
        TestFile("openai_server/validation/test_matched_stop.py", 60),
        TestFile("openai_server/validation/test_openai_server_ignore_eos.py", 85),
        TestFile("openai_server/validation/test_request_length_validation.py", 31),
59
60
61
        TestFile("quant/test_block_int8.py", 22),
        TestFile("quant/test_fp8_kernel.py", 8),
        TestFile("quant/test_int8_kernel.py", 8),
62
        TestFile("quant/test_triton_scaled_mm.py", 8),
63
        TestFile("quant/test_w8a8_quantization.py", 160),
64
        TestFile("quant/test_autoround.py", 60),
65
        TestFile("rl/test_fp32_lm_head.py", 30),
66
67
68
        TestFile("rl/test_update_weights_from_disk.py", 210),
        TestFile("rl/test_update_weights_from_tensor.py", 80),
        TestFile("test_abort.py", 190),
69
        TestFile("test_build_eagle_tree.py", 8),
70
        TestFile("test_chunked_prefill.py", 410),
71
        TestFile("test_create_kvindices.py", 2),
72
        TestFile("test_deterministic.py", 320),
73
        TestFile("test_eagle_infer_a.py", 370),
74
        TestFile("test_eagle_infer_b.py", 500),
75
        TestFile("test_eagle_infer_beta.py", 90),
76
        TestFile("test_constrained_decoding.py", 120),
Lianmin Zheng's avatar
Lianmin Zheng committed
77
        TestFile("test_eval_fp8_accuracy.py", 303),
78
79
80
        TestFile("test_fa3.py", 420),
        TestFile("test_flashmla.py", 230),
        TestFile("rotary_embedding/test_mrope.py", 10),
81
        TestFile("test_function_call_parser.py", 10),
82
83
        TestFile("test_fused_moe.py", 80),
        TestFile("test_gpt_oss_1gpu.py", 750),
84
        TestFile("test_harmony_parser.py", 20),
85
        TestFile("test_hidden_states.py", 55),
86
        TestFile("test_hybrid_attn_backend.py", 379),
Lianmin Zheng's avatar
Lianmin Zheng committed
87
        TestFile("test_input_embeddings.py", 38),
88
        TestFile("test_io_struct.py", 8),
89
        TestFile("test_jinja_template_utils.py", 1),
90
        TestFile("test_mamba_unittest.py", 4),
91
        TestFile("test_metrics.py", 32),
92
        TestFile("test_metrics_utils.py", 1),
93
        TestFile("test_mla.py", 180),
Lianmin Zheng's avatar
Lianmin Zheng committed
94
        TestFile("test_mla_deepseek_v3.py", 500),
95
96
        TestFile("test_mla_flashinfer.py", 302),
        TestFile("test_mla_fp8.py", 93),
97
        TestFile("test_mla_int8_deepseek_v3.py", 300),
98
        TestFile("test_modelopt_loader.py", 30),
99
        TestFile("test_multi_tokenizer.py", 230),
100
        TestFile("test_ngram_speculative_decoding.py", 290),
Lianmin Zheng's avatar
Lianmin Zheng committed
101
        TestFile("test_no_chunked_prefill.py", 108),
102
        TestFile("test_no_overlap_scheduler.py", 234),
103
        TestFile("test_original_logprobs.py", 41),
Lianmin Zheng's avatar
Lianmin Zheng committed
104
        TestFile("test_page_size.py", 60),
105
        TestFile("test_penalty.py", 82),
106
        TestFile("test_priority_scheduling.py", 130),
Lianmin Zheng's avatar
Lianmin Zheng committed
107
        TestFile("test_pytorch_sampling_backend.py", 66),
108
        TestFile("test_radix_attention.py", 105),
109
        TestFile("test_radix_cache_unit.py", 5),
110
        TestFile("test_reasoning_parser.py", 5),
111
        TestFile("test_request_queue_validation.py", 30),
112
        TestFile("test_retract_decode.py", 450),
113
        TestFile("test_score_api.py", 310),
Lianmin Zheng's avatar
Lianmin Zheng committed
114
        TestFile("test_server_args.py", 1),
115
        TestFile("test_speculative_registry.py", 1),
116
        TestFile("test_skip_tokenizer_init.py", 117),
Lianmin Zheng's avatar
Lianmin Zheng committed
117
        TestFile("test_srt_endpoint.py", 130),
118
        TestFile("test_srt_engine.py", 261),
119
        TestFile("test_standalone_speculative_decoding.py", 150),
120
        TestFile("test_start_profile.py", 60),
121
122
        TestFile("test_profile_merger.py", 60),
        TestFile("test_profile_merger_http_api.py", 15),
123
        TestFile("test_swa_unittest.py", 1),
Lianmin Zheng's avatar
Lianmin Zheng committed
124
        TestFile("test_torch_compile.py", 76),
125
        TestFile("test_torch_compile_moe.py", 210),
126
        TestFile("test_torch_native_attention_backend.py", 123),
Lianmin Zheng's avatar
Lianmin Zheng committed
127
        TestFile("test_torchao.py", 70),
128
        TestFile("test_triton_attention_kernels.py", 4),
129
        TestFile("test_triton_attention_backend.py", 150),
130
        TestFile("test_triton_attention_kernels.py", 4),
Lianmin Zheng's avatar
Lianmin Zheng committed
131
        TestFile("test_triton_moe_channel_fp8_kernel.py", 25),
132
        TestFile("test_triton_sliding_window.py", 100),
133
        TestFile("test_utils_update_weights.py", 48),
134
135
        TestFile("test_vision_chunked_prefill.py", 170),
        TestFile("test_vision_openai_server_a.py", 900),
136
        TestFile("test_vlm_input_format.py", 300),
137
138
        TestFile("test_modelopt_loader.py", 30),
        TestFile("test_modelopt_export.py", 30),
139
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
140
    "per-commit-2-gpu": [
141
        TestFile("ep/test_moe_ep.py", 140),
142
143
        TestFile("hicache/test_hicache_storage_3fs_backend.py", 200),
        TestFile("hicache/test_hicache_storage_file_backend.py", 200),
144
        TestFile("hicache/test_hicache_storage_mooncake_backend.py", 300),
145
        TestFile("layers/attention/mamba/test_mamba2_mixer.py", 50),
146
        TestFile("lora/test_lora_tp.py", 116),
147
        TestFile("models/test_glm4_moe_models.py", 100),
Ke Bao's avatar
Ke Bao committed
148
        TestFile("models/test_kimi_linear_models.py", 90),
149
        TestFile("rl/test_update_weights_from_distributed.py", 103),
Lianmin Zheng's avatar
Lianmin Zheng committed
150
        TestFile("test_data_parallelism.py", 73),
151
        TestFile("test_disaggregation_basic.py", 400),
152
        TestFile("test_dp_attention.py", 350),
153
        TestFile("test_load_weights_from_remote_instance.py", 72),
Lianmin Zheng's avatar
Lianmin Zheng committed
154
        TestFile("test_patch_torch.py", 19),
155
        TestFile("test_release_memory_occupation.py", 200),
156
        TestFile("test_eagle_dp_attention.py", 200),
Lianmin Zheng's avatar
Lianmin Zheng committed
157
158
    ],
    "per-commit-4-gpu": [
159
160
        TestFile("models/test_qwen3_next_models.py", 291),
        TestFile("test_disaggregation_dp_attention.py", 155),
Lianmin Zheng's avatar
Lianmin Zheng committed
161
162
        TestFile("test_gpt_oss_4gpu.py", 300),
        TestFile("test_local_attn.py", 411),
Lianmin Zheng's avatar
Lianmin Zheng committed
163
        TestFile("test_multi_instance_release_memory_occupation.py", 64),
164
        TestFile("test_pp_single_node.py", 481),
Lianmin Zheng's avatar
Lianmin Zheng committed
165
    ],
166
    "per-commit-8-gpu-h200": [
167
168
        TestFile("test_deepseek_v3_basic.py", 275),
        TestFile("test_deepseek_v3_mtp.py", 275),
169
        TestFile("test_disaggregation_hybrid_attention.py", 200),
Lianmin Zheng's avatar
Lianmin Zheng committed
170
    ],
171
    "per-commit-8-gpu-h20": [
172
        TestFile("quant/test_w4a8_deepseek_v3.py", 520),
173
174
175
        TestFile("test_disaggregation_different_tp.py", 600),
        TestFile("test_disaggregation_pp.py", 140),
    ],
Yineng Zhang's avatar
Yineng Zhang committed
176
    "per-commit-4-gpu-b200": [
177
        TestFile("test_deepseek_v3_fp4_4gpu.py", 3600),
Kangyan-Zhou's avatar
Kangyan-Zhou committed
178
179
        TestFile("test_flash_attention_4.py", 300),
        TestFile("test_gpt_oss_4gpu.py", 600),
180
        TestFile("test_llama31_fp4.py", 300),
Lianmin Zheng's avatar
Lianmin Zheng committed
181
    ],
182
    "per-commit-4-gpu-gb200": [
183
        TestFile("test_cutedsl_moe.py", 300),
184
        TestFile("test_deepseek_v3_fp4_4gpu.py", 3600),
185
186
        # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/12533
        # TestFile("test_deepseek_v3_cutedsl_4gpu.py", 3600),
187
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
188
    "per-commit-4-gpu-deepep": [
189
        TestFile("ep/test_deepep_small.py", 531),
190
        TestFile("ep/test_mooncake_ep_small.py", 450),
Lianmin Zheng's avatar
Lianmin Zheng committed
191
    ],
192
    "per-commit-8-gpu-h200-deepep": [
193
        TestFile("ep/test_deepep_large.py", 338),
Lianmin Zheng's avatar
Lianmin Zheng committed
194
    ],
195
    "per-commit-8-gpu-h200-deepseek-v32": [
196
        TestFile("test_deepseek_v32_basic.py", 275),
197
        TestFile("test_deepseek_v32_mtp.py", 275),
198
        TestFile("test_deepseek_v32_nsabackend.py", 600),
199
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
200
    "vllm_dependency_test": [
201
        TestFile("quant/test_awq.py", 163),
Lianmin Zheng's avatar
Lianmin Zheng committed
202
203
204
        TestFile("test_bnb.py", 5),
        TestFile("test_gptqmodel_dynamic.py", 102),
        TestFile("test_vllm_dependency.py", 185),
205
        TestFile("test_gguf.py", 96),
Lianmin Zheng's avatar
Lianmin Zheng committed
206
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
207
    # If the test cases take too long, considering adding them to nightly tests instead of per-commit tests
208
209
    "nightly-1-gpu": [
        TestFile("lora/test_lora_qwen3.py", 97),
210
        TestFile("lora/test_lora_qwen3_vl.py", 200),
211
212
213
214
215
216
217
        TestFile("lora/test_lora_radix_cache.py", 200),
        TestFile("lora/test_lora_eviction_policy.py", 200),
        TestFile("lora/test_lora_openai_api.py", 30),
        TestFile("openai_server/features/test_lora_openai_compatible.py", 150),
        TestFile("batch_invariant/test_batch_invariant_ops.py", 10),
        TestFile("test_deepseek_v3_deterministic.py", 240),
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
218
    "nightly-8-gpu": [],
219
220
221
    "__not_in_ci__": [
        TestFile("ascend/test_ascend_w8a8_quantization.py"),
        TestFile("cpu/test_comm.py"),
222
        TestFile("test_deepseek_v3_cutedsl_4gpu.py"),
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
        TestFile("entrypoints/http_server/test_abort_request.py"),
        TestFile("ep/test_deepep_internode.py"),
        TestFile("ep/test_deepep_intranode.py"),
        TestFile("ep/test_deepep_low_latency.py"),
        TestFile("ep/test_eplb.py"),
        TestFile("ep/test_hybrid_dp_ep_tp_mtp.py"),
        TestFile("ep/test_moe_deepep.py"),
        TestFile("ep/test_moe_deepep_eval_accuracy_large.py"),
        TestFile("function_call/test_unknown_tool_name.py"),
        TestFile("hicache/test_disaggregation_hicache.py"),
        TestFile("hicache/test_hicache_page.py"),
        TestFile("hicache/test_hicache_storage_benchmark.py"),
        TestFile("hicache/test_hicache_storage_e2e.py"),
        TestFile("layers/attention/nsa/test_act_quant_triton.py"),
        TestFile("lora/test_chunked_sgmv_backend.py"),
        TestFile("lora/test_lora_llama4.py"),
        TestFile("models/lora/test_lora.py"),
        TestFile("models/lora/test_lora_backend.py"),
        TestFile("models/lora/test_lora_cuda_graph.py"),
        TestFile("models/lora/test_lora_eviction.py"),
        TestFile("models/lora/test_lora_qwen3.py"),
        TestFile("models/lora/test_lora_tp.py"),
        TestFile("models/lora/test_lora_update.py"),
        TestFile("models/lora/test_multi_lora_backend.py"),
        TestFile("models/test_clip_models.py"),
        TestFile("models/test_dummy_grok_models.py"),
        TestFile("models/test_falcon_h1_models.py"),
        TestFile("models/test_gme_qwen_models.py"),
        TestFile("models/test_grok_models.py"),
        TestFile("models/test_llama4_models.py"),
        TestFile("models/test_mtp_models.py"),
        TestFile("models/test_unsloth_models.py"),
        TestFile("openai/test_server.py"),
        TestFile("openai_server/features/test_cache_report.py"),
        TestFile("openai_server/features/test_continuous_usage_stats.py"),
        TestFile("openai_server/features/test_structural_tag.py"),
        TestFile("quant/test_fp8_kvcache.py"),
        TestFile("rl/test_verl_engine_2_gpu.py"),
        TestFile("rl/test_verl_engine_4_gpu.py"),
        TestFile("test_ascend_attention_backend.py"),
        TestFile("test_ascend_mla_backend.py"),
        TestFile("test_ascend_mla_w8a8int8.py"),
        TestFile("test_ascend_tp1_bf16.py"),
        TestFile("test_ascend_tp2_bf16.py"),
        TestFile("test_ascend_w8a8_quantization.py"),
        TestFile("test_async_dynamic_batch_tokenizer.py"),
        TestFile("test_async_mm_data_processor.py"),
        TestFile("test_awq.py"),
        TestFile("test_awq_dequant.py"),
        TestFile("test_bench_one_batch.py"),
        TestFile("test_bench_serving.py"),
        TestFile("test_block_int8.py"),
        TestFile("test_cache_report.py"),
        TestFile("test_config_integration.py"),
        TestFile("test_cpp_radix_cache.py"),
        TestFile("test_cpu_graph.py"),
        TestFile("test_custom_allreduce.py"),
        TestFile("test_cutedsl_flashinfer_8gpu.py"),
        TestFile("test_deepep_internode.py"),
        TestFile("test_deepep_intranode.py"),
        TestFile("test_deepep_large.py"),
        TestFile("test_deepep_low_latency.py"),
        TestFile("test_deepep_small.py"),
        TestFile("test_deepseek_chat_templates.py"),
        TestFile("test_disaggregation.py"),
        TestFile("test_double_sparsity.py"),
        TestFile("test_embedding_openai_server.py"),
        TestFile("test_enable_thinking.py"),
        TestFile("test_eplb.py"),
        TestFile("test_eval_accuracy_large.py"),
        TestFile("test_expert_distribution.py"),
        TestFile("test_expert_location_updater.py"),
        TestFile("test_fim_completion.py"),
        TestFile("test_forward_split_prefill.py"),
        TestFile("test_fp8_kernel.py"),
        TestFile("test_fp8_kvcache.py"),
        TestFile("test_full_deepseek_v3.py"),
        TestFile("test_get_weights_by_name.py"),
        TestFile("test_gpt_oss_common.py"),
        TestFile("test_health_check.py"),
        TestFile("test_hicache.py"),
        TestFile("test_hicache_mla.py"),
        TestFile("test_hicache_page.py"),
        TestFile("test_hicache_storage.py"),
        TestFile("test_hybrid_dp_ep_tp_mtp.py"),
        TestFile("test_int4_kernel.py"),
        TestFile("test_int8_kernel.py"),
        TestFile("test_intel_amx_attention_backend.py"),
311
        TestFile("test_constrained_decoding.py"),
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
        TestFile("test_json_mode.py"),
        TestFile("test_kv_events.py"),
        TestFile("test_large_max_new_tokens.py"),
        TestFile("test_logprobs.py"),
        TestFile("test_lookahead_speculative_decoding.py"),
        TestFile("test_matched_stop.py"),
        TestFile("test_mla_tp.py"),
        TestFile("test_modelopt.py"),
        TestFile("test_modelopt_fp8kvcache.py"),
        TestFile("test_models_from_modelscope.py"),
        TestFile("test_moe_deepep.py"),
        TestFile("test_moe_deepep_eval_accuracy_large.py"),
        TestFile("test_moe_ep.py"),
        TestFile("test_moe_eval_accuracy_large.py"),
        TestFile("test_mscclpp.py"),
        TestFile("test_nightly_gsm8k_eval.py"),
        TestFile("test_nightly_text_models_gsm8k_eval.py"),
        TestFile("test_nightly_text_models_perf.py"),
        TestFile("test_nightly_vlms_mmmu_eval.py"),
        TestFile("test_nightly_vlms_perf.py"),
        TestFile("test_openai_adapter.py"),
        TestFile("test_openai_function_calling.py"),
        TestFile("test_openai_server.py"),
        TestFile("test_openai_server_hidden_states.py"),
        TestFile("test_piecewise_cuda_graph.py"),
        TestFile("test_quick_allreduce.py"),
        TestFile("test_reasoning_content.py"),
        TestFile("test_request_length_validation.py"),
        TestFile("test_sagemaker_server.py"),
        TestFile("test_schedule_policy.py"),
        TestFile("test_session_control.py"),
        TestFile("test_srt_engine_with_quant_args.py"),
        TestFile("test_tokenizer_batch_encode.py"),
        TestFile("test_tokenizer_manager.py"),
        TestFile("test_tool_choice.py"),
        TestFile("test_torch_flex_attention_backend.py"),
        TestFile("test_torch_tp.py"),
        TestFile("test_tracing.py"),
        TestFile("test_triton_attention_rocm_mla.py"),
        TestFile("test_triton_fused_moe.py"),
        TestFile("test_triton_moe_wna16.py"),
        TestFile("test_two_batch_overlap.py"),
        TestFile("test_update_weights_from_disk.py"),
        TestFile("test_update_weights_from_distributed.py"),
        TestFile("test_update_weights_from_tensor.py"),
        TestFile("test_verl_engine_2_gpu.py"),
        TestFile("test_verl_engine_4_gpu.py"),
        TestFile("test_verl_engine_server.py"),
        TestFile("test_vertex_endpoint.py"),
        TestFile("test_vision_openai_server_b.py"),
        TestFile("test_vision_openai_server_common.py"),
        TestFile("test_vlm_accuracy.py"),
        TestFile("test_w4a8.py"),
        TestFile("test_w8a8_quantization.py"),
        TestFile("test_wave_attention_backend.py"),
        TestFile("test_weight_version.py"),
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
369
370
371
}

# Add AMD tests
372
# NOTE: please sort the test cases alphabetically by the test file name
Lianmin Zheng's avatar
Lianmin Zheng committed
373
suite_amd = {
374
    "per-commit-amd": [
375
        TestFile("function_call/test_json_schema_constraint.py", 1),
376
377
378
        # TestFile("hicache/test_hicache.py", 116), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/12575
        # TestFile("hicache/test_hicache_mla.py", 127), # Disabled temporarily,  # Temporarily disabled, see https://github.com/sgl-project/sglang/issues/12574
        # TestFile("hicache/test_hicache_storage.py", 127), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/12575
379
        TestFile("lora/test_lora.py", 150),
380
381
        TestFile("lora/test_lora_backend.py", 99),
        TestFile("lora/test_lora_cuda_graph.py", 250),
382
        TestFile("lora/test_lora_eviction.py", 240),
383
        TestFile("lora/test_lora_qwen3.py", 97),
384
        TestFile("lora/test_multi_lora_backend.py", 60),
385
        TestFile("models/test_compressed_tensors_models.py", 42),
386
387
        TestFile("models/test_qwen_models.py", 82),
        TestFile("models/test_reward_models.py", 132),
388
        TestFile("models/test_transformers_models.py", 320),
389
390
391
        TestFile("models/test_vlm_models.py", 437),
        TestFile("openai_server/basic/test_openai_embedding.py", 141),
        TestFile("openai_server/basic/test_openai_server.py", 149),
392
393
394
395
        TestFile("openai_server/basic/test_protocol.py", 10),
        TestFile("openai_server/basic/test_serving_chat.py", 10),
        TestFile("openai_server/basic/test_serving_completions.py", 10),
        TestFile("openai_server/basic/test_serving_embedding.py", 10),
396
        TestFile("openai_server/features/test_enable_thinking.py", 70),
397
398
        TestFile("openai_server/features/test_json_mode.py", 120),
        TestFile("openai_server/features/test_openai_server_ebnf.py", 20),
399
        TestFile("openai_server/features/test_reasoning_content.py", 89),
400
        TestFile("openai_server/function_call/test_openai_function_calling.py", 60),
401
        TestFile("openai_server/function_call/test_tool_choice.py", 120),
402
        TestFile("openai_server/validation/test_large_max_new_tokens.py", 41),
403
404
        TestFile("openai_server/validation/test_matched_stop.py", 60),
        TestFile("openai_server/validation/test_openai_server_ignore_eos.py", 85),
405
        TestFile("openai_server/validation/test_request_length_validation.py", 31),
406
        TestFile("quant/test_awq_dequant.py", 2),
407
        TestFile("quant/test_block_int8.py", 22),
408
        TestFile("rl/test_update_weights_from_disk.py", 210),
409
        TestFile("test_abort.py", 51),
410
        TestFile("test_chunked_prefill.py", 410),
411
        TestFile("test_create_kvindices.py", 2),
412
413
        TestFile("test_eval_fp8_accuracy.py", 303),
        TestFile("test_function_call_parser.py", 10),
414
415
        TestFile("test_fused_moe.py", 30),
        TestFile("test_harmony_parser.py", 20),
416
        TestFile("test_input_embeddings.py", 38),
417
418
419
        TestFile("test_io_struct.py", 8),
        TestFile("test_jinja_template_utils.py", 1),
        TestFile("test_metrics.py", 32),
420
        TestFile("test_metrics_utils.py", 1),
421
        TestFile("test_mla.py", 242),
422
        # TestFile("test_mla_deepseek_v3.py", 221), # Temporarily disabled, see https://github.com/sgl-project/sglang/issues/12574
423
424
        TestFile("test_no_chunked_prefill.py", 108),
        TestFile("test_page_size.py", 60),
425
        TestFile("test_penalty.py", 180),
426
427
        TestFile("test_pytorch_sampling_backend.py", 66),
        TestFile("test_radix_attention.py", 105),
428
        TestFile("test_reasoning_parser.py", 5),
429
        TestFile("test_constrained_decoding.py", 120),
430
        TestFile("test_retract_decode.py", 450),
431
        TestFile("test_rope_rocm.py", 3),
432
433
        TestFile("test_server_args.py", 1),
        TestFile("test_skip_tokenizer_init.py", 117),
434
        TestFile("test_srt_endpoint.py", 130),
435
        TestFile("test_srt_engine.py", 261),
436
        TestFile("test_torch_compile.py", 169),
437
        TestFile("test_torch_compile_moe.py", 210),
438
        TestFile("test_torch_native_attention_backend.py", 123),
439
440
441
        # TestFile("test_triton_attention_kernels.py", 4),
        TestFile("test_triton_attention_backend.py", 150),
        TestFile("test_triton_sliding_window.py", 250),
442
        TestFile("test_wave_attention_kernels.py", 2),
443
        # Disabled temporarily
444
        # TestFile("test_vlm_input_format.py", 300),
445
446
447
448
449
        # TestFile("models/test_embedding_models.py", 73), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127
        # TestFile("openai_server/features/test_openai_server_hidden_states.py", 240),
        # TestFile("rl/test_update_weights_from_tensor.py", 48),
        # TestFile("test_no_overlap_scheduler.py", 234), # Disabled temporarily and track in #7703
        # TestFile("test_vision_chunked_prefill.py", 175), # Disabled temporarily and track in #7701
Sai Enduri's avatar
Sai Enduri committed
450
        # TestFile("test_wave_attention_backend.py", 150), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127
451
    ],
452
    "per-commit-amd-mi35x": [
453
        TestFile("test_gpt_oss_1gpu.py", 750),
454
        TestFile("test_mla.py", 242),
455
    ],
456
    "per-commit-2-gpu-amd": [
457
458
        TestFile("lora/test_lora_tp.py", 116),
        TestFile("rl/test_update_weights_from_distributed.py", 103),
459
        TestFile("test_data_parallelism.py", 73),
460
        TestFile("test_load_weights_from_remote_instance.py", 72),
Sai Enduri's avatar
Sai Enduri committed
461
        # TestFile("test_patch_torch.py", 19), # Disabled temporarily, see https://github.com/sgl-project/sglang/issues/11127
462
    ],
463
464
465
    "per-commit-4-gpu-amd": [
        TestFile("test_pp_single_node.py", 150),
    ],
466
    "per-commit-8-gpu-amd": [
467
468
        TestFile("test_deepseek_v3_basic.py", 275),
        TestFile("test_deepseek_v3_mtp.py", 275),
469
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
470
471
    "nightly-amd": [
        TestFile("test_nightly_gsm8k_eval_amd.py"),
fzyzcjy's avatar
fzyzcjy committed
472
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
473
474
475
}

# Add Intel Xeon tests
476
# NOTE: please sort the test cases alphabetically by the test file name
Lianmin Zheng's avatar
Lianmin Zheng committed
477
suite_xeon = {
478
479
    "per-commit-cpu": [
        TestFile("cpu/test_activation.py"),
480
        TestFile("cpu/test_binding.py"),
481
482
483
        TestFile("cpu/test_decode.py"),
        TestFile("cpu/test_extend.py"),
        TestFile("cpu/test_gemm.py"),
484
        TestFile("cpu/test_mla.py"),
485
486
487
        TestFile("cpu/test_moe.py"),
        TestFile("cpu/test_norm.py"),
        TestFile("cpu/test_qkv_proj_with_rope.py"),
488
        TestFile("cpu/test_rope.py"),
489
        TestFile("cpu/test_shared_expert.py"),
490
        TestFile("cpu/test_topk.py"),
491
492
493
494
        TestFile("cpu/test_cpu_graph.py"),
        TestFile("cpu/test_intel_amx_attention_backend_a.py"),
        TestFile("cpu/test_intel_amx_attention_backend_b.py"),
        TestFile("cpu/test_intel_amx_attention_backend_c.py"),
495
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
496
497
}

498
499
500
501
502
503
504
# Add Intel XPU tests
suite_xpu = {
    "per-commit-xpu": [
        TestFile("xpu/test_intel_xpu_backend.py"),
    ],
}

Lianmin Zheng's avatar
Lianmin Zheng committed
505
# Add Ascend NPU tests
506
# NOTE: please sort the test cases alphabetically by the test file name
Lianmin Zheng's avatar
Lianmin Zheng committed
507
508
suite_ascend = {
    "per-commit-1-ascend-npu": [
509
        TestFile("ascend/test_ascend_graph_tp1_bf16.py", 400),
510
        TestFile("ascend/test_ascend_tp1_bf16.py", 400),
511
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
512
    "per-commit-2-ascend-npu": [
513
        TestFile("ascend/test_ascend_graph_tp2_bf16.py", 400),
514
        TestFile("ascend/test_ascend_mla_fia_w8a8int8.py", 400),
515
516
        TestFile("ascend/test_ascend_tp2_bf16.py", 400),
        TestFile("ascend/test_ascend_tp2_fia_bf16.py", 400),
517
    ],
Lianmin Zheng's avatar
Lianmin Zheng committed
518
    "per-commit-4-ascend-npu": [
519
        TestFile("ascend/test_ascend_mla_w8a8int8.py", 400),
520
        TestFile("ascend/test_ascend_tp4_bf16.py", 400),
521
    ],
522
523
    "per-commit-16-ascend-a3": [
        TestFile("ascend/test_ascend_deepep.py", 400),
524
        TestFile("ascend/test_ascend_deepseek_mtp.py", 400),
525
    ],
526
527
}

Lianmin Zheng's avatar
Lianmin Zheng committed
528
529
530
suites.update(suite_amd)
suites.update(suite_xeon)
suites.update(suite_ascend)
531
suites.update(suite_xpu)
Lianmin Zheng's avatar
Lianmin Zheng committed
532

Lianmin Zheng's avatar
Lianmin Zheng committed
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576

def auto_partition(files, rank, size):
    """
    Partition files into size sublists with approximately equal sums of estimated times
    using stable sorting, and return the partition for the specified rank.

    Args:
        files (list): List of file objects with estimated_time attribute
        rank (int): Index of the partition to return (0 to size-1)
        size (int): Number of partitions

    Returns:
        list: List of file objects in the specified rank's partition
    """
    weights = [f.estimated_time for f in files]

    if not weights or size <= 0 or size > len(weights):
        return []

    # Create list of (weight, original_index) tuples
    # Using negative index as secondary key to maintain original order for equal weights
    indexed_weights = [(w, -i) for i, w in enumerate(weights)]
    # Stable sort in descending order by weight
    # If weights are equal, larger (negative) index comes first (i.e., earlier original position)
    indexed_weights = sorted(indexed_weights, reverse=True)

    # Extract original indices (negate back to positive)
    indexed_weights = [(w, -i) for w, i in indexed_weights]

    # Initialize partitions and their sums
    partitions = [[] for _ in range(size)]
    sums = [0.0] * size

    # Greedy approach: assign each weight to partition with smallest current sum
    for weight, idx in indexed_weights:
        # Find partition with minimum sum
        min_sum_idx = sums.index(min(sums))
        partitions[min_sum_idx].append(idx)
        sums[min_sum_idx] += weight

    # Return the files corresponding to the indices in the specified rank's partition
    indices = partitions[rank]
    return [files[i] for i in indices]

577

578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
def _sanity_check_suites(suites):
    dir_base = Path(__file__).parent
    disk_files = set(
        [
            str(x.relative_to(dir_base))
            for x in dir_base.glob("**/*.py")
            if x.name.startswith("test_")
        ]
    )

    suite_files = set(
        [test_file.name for _, suite in suites.items() for test_file in suite]
    )

    missing_files = sorted(list(disk_files - suite_files))
    missing_text = "\n".join(f'TestFile("{x}"),' for x in missing_files)
    assert len(missing_files) == 0, (
        f"Some test files are not in test suite. "
        f"If this is intentional, please add the following to `not_in_ci` section:\n"
        f"{missing_text}"
    )


601
602
603
604
605
if __name__ == "__main__":
    arg_parser = argparse.ArgumentParser()
    arg_parser.add_argument(
        "--timeout-per-file",
        type=int,
606
        default=1200,
607
608
609
610
611
612
613
614
615
        help="The time limit for running one file in seconds.",
    )
    arg_parser.add_argument(
        "--suite",
        type=str,
        default=list(suites.keys())[0],
        choices=list(suites.keys()) + ["all"],
        help="The suite to run",
    )
616
617
618
619
620
621
622
623
624
625
626
627
    arg_parser.add_argument(
        "--range-begin",
        type=int,
        default=0,
        help="The begin index of the range of the files to run.",
    )
    arg_parser.add_argument(
        "--range-end",
        type=int,
        default=None,
        help="The end index of the range of the files to run.",
    )
Lianmin Zheng's avatar
Lianmin Zheng committed
628
629
630
631
632
633
634
635
636
637
    arg_parser.add_argument(
        "--auto-partition-id",
        type=int,
        help="Use auto load balancing. The part id.",
    )
    arg_parser.add_argument(
        "--auto-partition-size",
        type=int,
        help="Use auto load balancing. The number of parts.",
    )
638
    args = arg_parser.parse_args()
Lianmin Zheng's avatar
Lianmin Zheng committed
639
    print(f"{args=}")
640

641
642
    _sanity_check_suites(suites)

643
644
645
646
647
    if args.suite == "all":
        files = glob.glob("**/test_*.py", recursive=True)
    else:
        files = suites[args.suite]

Lianmin Zheng's avatar
Lianmin Zheng committed
648
649
650
651
    if args.auto_partition_size:
        files = auto_partition(files, args.auto_partition_id, args.auto_partition_size)
    else:
        files = files[args.range_begin : args.range_end]
652

Lianmin Zheng's avatar
Lianmin Zheng committed
653
    print("The running tests are ", [f.name for f in files])
654

655
656
    exit_code = run_unittest_files(files, args.timeout_per_file)
    exit(exit_code)