test_profile_sla_dryrun.py 19 KB
Newer Older
1
2
3
4
5
6
7
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""
Test suite for profile_sla dry-run functionality.

This test ensures that the profile_sla script can successfully run in dry-run mode
8
for vllm, sglang, and trtllm backends with their respective disagg.yaml configurations.
9
10
11
12
"""

import sys
from pathlib import Path
13
from unittest.mock import patch
14
15
16
17
18
19
20
21

import pytest

# Add the project root to sys.path to enable imports
project_root = Path(__file__).parent.parent.parent
sys.path.insert(0, str(project_root))

from benchmarks.profiler.profile_sla import run_profile  # noqa: E402
22
from benchmarks.profiler.utils.model_info import ModelInfo  # noqa: E402
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
from benchmarks.profiler.utils.search_space_autogen import (  # noqa: E402
    auto_generate_search_space,
)


# Override the logger fixture from conftest.py to prevent directory creation
@pytest.fixture(autouse=True)
def logger(request):
    """Override the logger fixture to prevent test directory creation.

    This replaces the logger fixture from tests/conftest.py that creates
    directories named after each test.
    """
    # Simply do nothing - no directories created, no file handlers added
    yield
38
39
40
41
42
43


class TestProfileSLADryRun:
    """Test class for profile_sla dry-run functionality."""

    @pytest.fixture
44
    def vllm_args(self, request):
45
46
47
        """Create arguments for vllm backend dry-run test."""

        class Args:
48
49
            def __init__(self):
                self.backend = "vllm"
50
                self.config = "examples/backends/vllm/deploy/disagg.yaml"
51
52
53
                # Use unique output directory per test for parallel execution
                self.output_dir = f"/tmp/test_profiling_results_{request.node.name}"
                self.namespace = f"test-namespace-{request.node.name}"
54
55
                self.model = ""
                self.dgd_image = ""
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
                self.min_num_gpus_per_engine = 1
                self.max_num_gpus_per_engine = 8
                self.skip_existing_results = False
                self.force_rerun = False
                self.isl = 3000
                self.osl = 500
                self.ttft = 50
                self.itl = 10
                self.max_context_length = 16384
                self.prefill_interpolation_granularity = 16
                self.decode_interpolation_granularity = 6
                self.service_name = ""
                self.dry_run = True
                self.use_ai_configurator = False
                self.aic_system = None
71
                self.aic_hf_id = None
72
73
                self.aic_backend = ""
                self.aic_backend_version = None
74
                self.num_gpus_per_node = 8
75
                self.deploy_after_profile = False
76
                self.pick_with_webui = False
77
78
79
80
81
82
83
                # Provide minimal model_info to avoid HF queries
                self.model_info = ModelInfo(
                    model_size=16384.0,
                    architecture="TestArchitecture",
                    is_moe=False,
                    max_context_length=self.max_context_length,
                )
84
85
86
87

        return Args()

    @pytest.fixture
88
    def sglang_args(self, request):
89
90
91
        """Create arguments for sglang backend dry-run test."""

        class Args:
92
93
            def __init__(self):
                self.backend = "sglang"
94
                self.config = "examples/backends/sglang/deploy/disagg.yaml"
95
96
97
                # Use unique output directory per test for parallel execution
                self.output_dir = f"/tmp/test_profiling_results_{request.node.name}"
                self.namespace = f"test-namespace-{request.node.name}"
98
99
                self.model = ""
                self.dgd_image = ""
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
                self.min_num_gpus_per_engine = 1
                self.max_num_gpus_per_engine = 8
                self.skip_existing_results = False
                self.force_rerun = False
                self.isl = 3000
                self.osl = 500
                self.ttft = 50
                self.itl = 10
                self.max_context_length = 16384
                self.prefill_interpolation_granularity = 16
                self.decode_interpolation_granularity = 6
                self.service_name = ""
                self.dry_run = True
                self.use_ai_configurator = False
                self.aic_system = None
115
                self.aic_hf_id = None
116
117
                self.aic_backend = ""
                self.aic_backend_version = None
118
                self.num_gpus_per_node = 8
119
                self.deploy_after_profile = False
120
                self.pick_with_webui = False
121
122
123
124
125
126
                self.model_info = ModelInfo(
                    model_size=16384.0,
                    architecture="TestArchitecture",
                    is_moe=False,
                    max_context_length=self.max_context_length,
                )
127
128
129
130

        return Args()

    @pytest.mark.pre_merge
131
    @pytest.mark.parallel
132
    @pytest.mark.asyncio
133
134
135
    @pytest.mark.gpu_0
    @pytest.mark.integration
    @pytest.mark.vllm
136
137
138
139
140
141
    async def test_vllm_dryrun(self, vllm_args):
        """Test that profile_sla dry-run works for vllm backend with disagg.yaml config."""
        # Run the profile in dry-run mode - should complete without errors
        await run_profile(vllm_args)

    @pytest.mark.pre_merge
142
    @pytest.mark.parallel
143
    @pytest.mark.asyncio
144
145
146
    @pytest.mark.gpu_0
    @pytest.mark.integration
    @pytest.mark.sglang
147
148
149
150
    async def test_sglang_dryrun(self, sglang_args):
        """Test that profile_sla dry-run works for sglang backend with disagg.yaml config."""
        # Run the profile in dry-run mode - should complete without errors
        await run_profile(sglang_args)
151
152

    @pytest.fixture
153
    def trtllm_args(self, request):
154
155
156
        """Create arguments for trtllm backend dry-run test."""

        class Args:
157
158
            def __init__(self):
                self.backend = "trtllm"
159
                self.config = "examples/backends/trtllm/deploy/disagg.yaml"
160
161
162
                # Use unique output directory per test for parallel execution
                self.output_dir = f"/tmp/test_profiling_results_{request.node.name}"
                self.namespace = f"test-namespace-{request.node.name}"
163
164
                self.model = ""
                self.dgd_image = ""
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
                self.min_num_gpus_per_engine = 1
                self.max_num_gpus_per_engine = 8
                self.skip_existing_results = False
                self.force_rerun = False
                self.isl = 3000
                self.osl = 500
                self.ttft = 50
                self.itl = 10
                self.max_context_length = 16384
                self.prefill_interpolation_granularity = 16
                self.decode_interpolation_granularity = 6
                self.service_name = ""
                self.dry_run = True
                self.use_ai_configurator = False
                self.aic_system = None
180
                self.aic_hf_id = None
181
182
                self.aic_backend = ""
                self.aic_backend_version = None
183
                self.num_gpus_per_node = 8
184
                self.deploy_after_profile = False
185
                self.pick_with_webui = False
186
187
188
189
190
191
                self.model_info = ModelInfo(
                    model_size=16384.0,
                    architecture="TestArchitecture",
                    is_moe=False,
                    max_context_length=self.max_context_length,
                )
192
193
194
195

        return Args()

    @pytest.mark.pre_merge
196
    @pytest.mark.parallel
197
    @pytest.mark.asyncio
198
199
200
    @pytest.mark.gpu_0
    @pytest.mark.integration
    @pytest.mark.trtllm
201
202
203
204
    async def test_trtllm_dryrun(self, trtllm_args):
        """Test that profile_sla dry-run works for trtllm backend with disagg.yaml config."""
        # Run the profile in dry-run mode - should complete without errors
        await run_profile(trtllm_args)
205
206

    @pytest.fixture
207
    def sglang_moe_args(self, request):
208
209
210
211
212
        """Create arguments for trtllm backend dry-run test."""

        class Args:
            def __init__(self):
                self.backend = "sglang"
213
                self.config = "recipes/deepseek-r1/sglang/disagg-16gpu/deploy.yaml"
214
215
216
                # Use unique output directory per test for parallel execution
                self.output_dir = f"/tmp/test_profiling_results_{request.node.name}"
                self.namespace = f"test-namespace-{request.node.name}"
217
218
                self.model = ""
                self.dgd_image = ""
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
                self.min_num_gpus_per_engine = 8
                self.max_num_gpus_per_engine = 32
                self.skip_existing_results = False
                self.force_rerun = False
                self.isl = 3000
                self.osl = 500
                self.ttft = 50
                self.itl = 10
                self.max_context_length = 16384
                self.prefill_interpolation_granularity = 16
                self.decode_interpolation_granularity = 6
                self.service_name = ""
                self.dry_run = True
                self.use_ai_configurator = False
                self.aic_system = None
234
                self.aic_hf_id = None
235
236
                self.aic_backend = ""
                self.aic_backend_version = None
237
                self.num_gpus_per_node = 8
238
                self.deploy_after_profile = False
239
                self.pick_with_webui = False
240
241
242
243
244
245
246
                self.model_info = ModelInfo(
                    model_size=65536.0,
                    architecture="TestMoEArchitecture",
                    is_moe=True,
                    max_context_length=self.max_context_length,
                    num_experts=16,
                )
247
248
249
250

        return Args()

    @pytest.mark.pre_merge
251
    @pytest.mark.parallel
252
    @pytest.mark.asyncio
253
254
255
    @pytest.mark.gpu_0
    @pytest.mark.integration
    @pytest.mark.sglang
256
257
258
259
    async def test_sglang_moe_dryrun(self, sglang_moe_args):
        """Test that profile_sla dry-run works for sglang backend with MoE config."""
        # Run the profile in dry-run mode - should complete without errors
        await run_profile(sglang_moe_args)
260
261
262
263
264
265
266
267
268
269
270
271
272
273

    # Example tests with mocked GPU inventory
    @pytest.fixture
    def mock_h100_gpu_info(self):
        """Mock GPU info for H100 80GB cluster."""
        return {
            "gpus_per_node": 8,
            "model": "h100_sxm",
            "vram": 81920,  # 80GB in MiB
        }

    @pytest.fixture
    def mock_model_info(self):
        """Mock model info for DeepSeek-R1-Distill-Llama-8B."""
274
275
276
277
278
279
        return ModelInfo(
            model_size=16384.0,  # 16GB model in MiB
            architecture="LlamaForCausalLM",
            is_moe=False,
            max_context_length=16384,
        )
280
281

    @pytest.fixture
282
    def vllm_args_with_model_autogen(self, request):
283
284
285
286
287
288
        """Create arguments for vllm backend with model-based search space autogeneration."""

        class Args:
            def __init__(self):
                self.backend = "vllm"
                self.config = ""
289
290
291
                # Use unique output directory per test for parallel execution
                self.output_dir = f"/tmp/test_profiling_results_{request.node.name}"
                self.namespace = f"test-namespace-{request.node.name}"
292
                self.model = "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"  # Specify model for autogen
293
                self.dgd_image = ""
294
295
296
                # Set to 0 to trigger auto-generation path
                self.min_num_gpus_per_engine = 0
                self.max_num_gpus_per_engine = 0
297
298
299
300
301
302
303
304
305
306
307
308
309
                self.skip_existing_results = False
                self.force_rerun = False
                self.isl = 3000
                self.osl = 500
                self.ttft = 50
                self.itl = 10
                self.max_context_length = 0
                self.prefill_interpolation_granularity = 16
                self.decode_interpolation_granularity = 6
                self.service_name = ""
                self.dry_run = True
                self.use_ai_configurator = False
                self.aic_system = None
310
                self.aic_hf_id = None
311
312
                self.aic_backend = ""
                self.aic_backend_version = None
313
314
                # Set to 0 to trigger auto-generation path
                self.num_gpus_per_node = 0
315
                self.deploy_after_profile = False
316
                self.pick_with_webui = False
317
                self.enable_gpu_discovery = True
318
319
320
321

        return Args()

    @pytest.mark.pre_merge
322
    @pytest.mark.parallel
323
    @pytest.mark.asyncio
324
325
326
    @pytest.mark.integration
    @pytest.mark.gpu_0
    @pytest.mark.vllm
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
    @patch("benchmarks.profiler.utils.search_space_autogen.get_gpu_summary")
    @patch("benchmarks.profiler.utils.search_space_autogen.get_model_info")
    async def test_profile_with_autogen_search_space_h100(
        self,
        mock_get_model_info,
        mock_get_gpu_summary,
        vllm_args_with_model_autogen,
        mock_h100_gpu_info,
        mock_model_info,
    ):
        """Test profile_sla with auto-generated search space on mocked H100 cluster.

        This test demonstrates how search space is auto-generated based on model
        size and available GPU memory.
        """
        # Configure the mocks to return the appropriate info
        mock_get_model_info.return_value = mock_model_info
        mock_get_gpu_summary.return_value = mock_h100_gpu_info

        # Run the profile - the search space will be auto-generated
        # based on the model and mocked GPU info
        auto_generate_search_space(vllm_args_with_model_autogen)
        await run_profile(vllm_args_with_model_autogen)

    @pytest.fixture
352
    def sglang_args_with_model_autogen(self, request):
353
354
355
356
357
358
        """Create arguments for sglang backend with model-based search space autogeneration."""

        class Args:
            def __init__(self):
                self.backend = "sglang"
                self.config = ""
359
360
361
                # Use unique output directory per test for parallel execution
                self.output_dir = f"/tmp/test_profiling_results_{request.node.name}"
                self.namespace = f"test-namespace-{request.node.name}"
362
                self.model = "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"  # Specify model for autogen
363
                self.dgd_image = ""
364
365
                self.min_num_gpus_per_engine = 0
                self.max_num_gpus_per_engine = 0
366
367
368
369
370
371
372
373
374
375
376
377
378
                self.skip_existing_results = False
                self.force_rerun = False
                self.isl = 3000
                self.osl = 500
                self.ttft = 50
                self.itl = 10
                self.max_context_length = 0
                self.prefill_interpolation_granularity = 16
                self.decode_interpolation_granularity = 6
                self.service_name = ""
                self.dry_run = True
                self.use_ai_configurator = False
                self.aic_system = None
379
                self.aic_hf_id = None
380
381
                self.aic_backend = ""
                self.aic_backend_version = None
382
                self.num_gpus_per_node = 0
383
                self.deploy_after_profile = False
384
                self.pick_with_webui = False
385
                self.enable_gpu_discovery = True
386
387
388
389

        return Args()

    @pytest.mark.pre_merge
390
    @pytest.mark.parallel
391
    @pytest.mark.asyncio
392
393
394
    @pytest.mark.gpu_0
    @pytest.mark.integration
    @pytest.mark.sglang
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
    @patch("benchmarks.profiler.utils.search_space_autogen.get_gpu_summary")
    @patch("benchmarks.profiler.utils.search_space_autogen.get_model_info")
    async def test_sglang_profile_with_autogen_search_space_h100(
        self,
        mock_get_model_info,
        mock_get_gpu_summary,
        sglang_args_with_model_autogen,
        mock_h100_gpu_info,
        mock_model_info,
    ):
        """Test profile_sla with auto-generated search space for sglang on mocked H100 cluster.

        This test demonstrates how search space is auto-generated based on model
        size and available GPU memory for sglang backend.
        """
        # Configure the mocks to return the appropriate info
        mock_get_model_info.return_value = mock_model_info
        mock_get_gpu_summary.return_value = mock_h100_gpu_info

        # Run the profile - the search space will be auto-generated
        # based on the model and mocked GPU info
        auto_generate_search_space(sglang_args_with_model_autogen)
        await run_profile(sglang_args_with_model_autogen)

    @pytest.fixture
420
    def trtllm_args_with_model_autogen(self, request):
421
422
423
424
425
426
        """Create arguments for trtllm backend with model-based search space autogeneration."""

        class Args:
            def __init__(self):
                self.backend = "trtllm"
                self.config = ""
427
428
429
                # Use unique output directory per test for parallel execution
                self.output_dir = f"/tmp/test_profiling_results_{request.node.name}"
                self.namespace = f"test-namespace-{request.node.name}"
430
                self.model = "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"  # Specify model for autogen
431
                self.dgd_image = ""
432
433
                self.min_num_gpus_per_engine = 0
                self.max_num_gpus_per_engine = 0
434
435
436
437
438
439
440
441
442
443
444
445
446
                self.skip_existing_results = False
                self.force_rerun = False
                self.isl = 3000
                self.osl = 500
                self.ttft = 50
                self.itl = 10
                self.max_context_length = 0
                self.prefill_interpolation_granularity = 16
                self.decode_interpolation_granularity = 6
                self.service_name = ""
                self.dry_run = True
                self.use_ai_configurator = False
                self.aic_system = None
447
                self.aic_hf_id = None
448
449
                self.aic_backend = ""
                self.aic_backend_version = None
450
                self.num_gpus_per_node = 0
451
                self.deploy_after_profile = False
452
                self.pick_with_webui = False
453
                self.enable_gpu_discovery = True
454
455
456
457

        return Args()

    @pytest.mark.pre_merge
458
    @pytest.mark.parallel
459
    @pytest.mark.asyncio
460
461
462
    @pytest.mark.gpu_0
    @pytest.mark.integration
    @pytest.mark.trtllm
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
    @patch("benchmarks.profiler.utils.search_space_autogen.get_gpu_summary")
    @patch("benchmarks.profiler.utils.search_space_autogen.get_model_info")
    async def test_trtllm_profile_with_autogen_search_space_h100(
        self,
        mock_get_model_info,
        mock_get_gpu_summary,
        trtllm_args_with_model_autogen,
        mock_h100_gpu_info,
        mock_model_info,
    ):
        """Test profile_sla with auto-generated search space for trtllm on mocked H100 cluster.

        This test demonstrates how search space is auto-generated based on model
        size and available GPU memory for trtllm backend.
        """
        # Configure the mocks to return the appropriate info
        mock_get_model_info.return_value = mock_model_info
        mock_get_gpu_summary.return_value = mock_h100_gpu_info

        # Run the profile - the search space will be auto-generated
        # based on the model and mocked GPU info
        auto_generate_search_space(trtllm_args_with_model_autogen)
        await run_profile(trtllm_args_with_model_autogen)