Unverified Commit d7b20dd6 authored by kushanam's avatar kushanam Committed by GitHub
Browse files

chore: Initial support for input config files (#10534)


Co-authored-by: default avatarroot <root@umbriel-b200-017.ipp4a1.colossus.nvidia.com>
Co-authored-by: default avatarYineng Zhang <me@zhyncs.com>
parent c3faf2d6
...@@ -8,6 +8,23 @@ You can find all arguments by `python3 -m sglang.launch_server --help` ...@@ -8,6 +8,23 @@ You can find all arguments by `python3 -m sglang.launch_server --help`
## Common launch commands ## Common launch commands
- To use a configuration file, create a YAML file with your server arguments and specify it with `--config`. CLI arguments will override config file values.
```bash
# Create config.yaml
cat > config.yaml << EOF
model-path: meta-llama/Meta-Llama-3-8B-Instruct
host: 0.0.0.0
port: 30000
tensor-parallel-size: 2
enable-metrics: true
log-requests: true
EOF
# Launch server with config file
python -m sglang.launch_server --config config.yaml
```
- To enable multi-GPU tensor parallelism, add `--tp 2`. If it reports the error "peer access is not supported between these two devices", add `--enable-p2p-check` to the server launch command. - To enable multi-GPU tensor parallelism, add `--tp 2`. If it reports the error "peer access is not supported between these two devices", add `--enable-p2p-check` to the server launch command.
```bash ```bash
...@@ -65,6 +82,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s ...@@ -65,6 +82,7 @@ Please consult the documentation below and [server_args.py](https://github.com/s
| Arguments | Description | Defaults | | Arguments | Description | Defaults |
|-----------|-------------|----------| |-----------|-------------|----------|
| `--config` | Path to a YAML configuration file containing server arguments. Arguments in the config file will be merged with command-line arguments, with CLI arguments taking precedence. | None |
| `--model-path` | The path of the model weights. This can be a local folder or a Hugging Face repo ID. | None | | `--model-path` | The path of the model weights. This can be a local folder or a Hugging Face repo ID. | None |
| `--tokenizer-path` | The path of the tokenizer. | None | | `--tokenizer-path` | The path of the tokenizer. | None |
| `--tokenizer-mode` | Tokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer. | auto | | `--tokenizer-mode` | Tokenizer mode. 'auto' will use the fast tokenizer if available, and 'slow' will always use the slow tokenizer. | auto |
......
...@@ -2659,6 +2659,13 @@ class ServerArgs: ...@@ -2659,6 +2659,13 @@ class ServerArgs:
help="NOTE: --enable-flashinfer-mxfp4-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_mxfp4' instead.", help="NOTE: --enable-flashinfer-mxfp4-moe is deprecated. Please set `--moe-runner-backend` to 'flashinfer_mxfp4' instead.",
) )
# Configuration file support
parser.add_argument(
"--config",
type=str,
help="Read CLI options from a config file. Must be a YAML file with configuration options.",
)
@classmethod @classmethod
def from_cli_args(cls, args: argparse.Namespace): def from_cli_args(cls, args: argparse.Namespace):
args.tp_size = args.tensor_parallel_size args.tp_size = args.tensor_parallel_size
...@@ -2939,6 +2946,26 @@ def prepare_server_args(argv: List[str]) -> ServerArgs: ...@@ -2939,6 +2946,26 @@ def prepare_server_args(argv: List[str]) -> ServerArgs:
Returns: Returns:
The server arguments. The server arguments.
""" """
# Import here to avoid circular imports
from sglang.srt.server_args_config_parser import ConfigArgumentMerger
# Check for config file and merge arguments if present
if "--config" in argv:
# Extract boolean actions from the parser to handle them correctly
parser = argparse.ArgumentParser()
ServerArgs.add_cli_args(parser)
# Get boolean action destinations
boolean_actions = []
for action in parser._actions:
if hasattr(action, "dest") and hasattr(action, "action"):
if action.action in ["store_true", "store_false"]:
boolean_actions.append(action.dest)
# Merge config file arguments with CLI arguments
config_merger = ConfigArgumentMerger(boolean_actions=boolean_actions)
argv = config_merger.merge_config_with_args(argv)
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
ServerArgs.add_cli_args(parser) ServerArgs.add_cli_args(parser)
raw_args = parser.parse_args(argv) raw_args = parser.parse_args(argv)
......
"""
Configuration argument parser for command-line applications.
Handles merging of YAML configuration files with command-line arguments.
"""
import logging
from pathlib import Path
from typing import Any, Dict, List, Union
import yaml
logger = logging.getLogger(__name__)
class ConfigArgumentMerger:
"""Handles merging of configuration file arguments with command-line arguments."""
def __init__(self, boolean_actions: List[str] = None):
"""Initialize with list of boolean action destinations."""
self.boolean_actions = boolean_actions or []
def merge_config_with_args(self, cli_args: List[str]) -> List[str]:
"""
Merge configuration file arguments with command-line arguments.
Configuration arguments are inserted after the subcommand to maintain
proper precedence: CLI > Config > Defaults
Args:
cli_args: List of command-line arguments
Returns:
Merged argument list with config values inserted
Raises:
ValueError: If multiple config files specified or no config file provided
"""
config_file_path = self._extract_config_file_path(cli_args)
if not config_file_path:
return cli_args
config_args = self._parse_yaml_config(config_file_path)
return self._insert_config_args(cli_args, config_args, config_file_path)
def _extract_config_file_path(self, args: List[str]) -> str:
"""Extract the config file path from arguments."""
config_indices = [i for i, arg in enumerate(args) if arg == "--config"]
if len(config_indices) > 1:
raise ValueError("Multiple config files specified! Only one allowed.")
if not config_indices:
return None
config_index = config_indices[0]
if config_index == len(args) - 1:
raise ValueError("No config file specified after --config flag!")
return args[config_index + 1]
def _insert_config_args(
self, cli_args: List[str], config_args: List[str], config_file_path: str
) -> List[str]:
"""Insert configuration arguments into the CLI argument list."""
config_index = cli_args.index("--config")
# Split arguments around config file
before_config = cli_args[:config_index]
after_config = cli_args[config_index + 2 :] # Skip --config and file path
# Simple merge: config args + CLI args
return config_args + before_config + after_config
def _parse_yaml_config(self, file_path: str) -> List[str]:
"""
Parse YAML configuration file and convert to argument list.
Args:
file_path: Path to the YAML configuration file
Returns:
List of arguments in format ['--key', 'value', ...]
Raises:
ValueError: If file is not YAML or cannot be read
"""
self._validate_yaml_file(file_path)
try:
with open(file_path, "r") as file:
config_data = yaml.safe_load(file)
except Exception as e:
logger.error(f"Failed to read config file {file_path}: {e}")
raise
# Handle empty files or None content
if config_data is None:
config_data = {}
if not isinstance(config_data, dict):
raise ValueError("Config file must contain a dictionary at root level")
return self._convert_config_to_args(config_data)
def _validate_yaml_file(self, file_path: str) -> None:
"""Validate that the file is a YAML file."""
path = Path(file_path)
if path.suffix.lower() not in [".yaml", ".yml"]:
raise ValueError(f"Config file must be YAML format, got: {path.suffix}")
if not path.exists():
raise ValueError(f"Config file not found: {file_path}")
def _convert_config_to_args(self, config: Dict[str, Any]) -> List[str]:
"""Convert configuration dictionary to argument list."""
args = []
for key, value in config.items():
if isinstance(value, bool):
self._add_boolean_arg(args, key, value)
elif isinstance(value, list):
self._add_list_arg(args, key, value)
else:
self._add_scalar_arg(args, key, value)
return args
def _add_boolean_arg(self, args: List[str], key: str, value: bool) -> None:
"""Add boolean argument to the list."""
if key in self.boolean_actions:
# For boolean actions, always add the flag and value
args.extend([f"--{key}", str(value).lower()])
else:
# For regular booleans, only add flag if True
if value:
args.append(f"--{key}")
def _add_list_arg(self, args: List[str], key: str, value: List[Any]) -> None:
"""Add list argument to the list."""
if value: # Only add if list is not empty
args.append(f"--{key}")
args.extend(str(item) for item in value)
def _add_scalar_arg(self, args: List[str], key: str, value: Any) -> None:
"""Add scalar argument to the list."""
args.extend([f"--{key}", str(value)])
"""
Test script to verify SGLang config file integration.
"""
import os
import tempfile
from pathlib import Path
import pytest
import yaml
from sglang.srt.server_args import prepare_server_args
from sglang.srt.server_args_config_parser import ConfigArgumentMerger
@pytest.fixture
def merger():
"""Fixture providing a ConfigArgumentMerger instance."""
return ConfigArgumentMerger()
def test_server_args_config_parser(merger):
"""Test the config parser functionality."""
# Create a temporary config file
config_data = {
"model-path": "microsoft/DialoGPT-medium",
"host": "0.0.0.0",
"port": 30000,
"tensor-parallel-size": 2,
"trust-remote-code": False,
"enable-metrics": True,
"stream-output": True,
"skip-server-warmup": False,
"log-requests": True,
"show-time-cost": True,
"is-embedding": False,
}
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
yaml.dump(config_data, f)
config_file = f.name
try:
# Test config parser directly
config_args = merger._parse_yaml_config(config_file)
# Test merging with CLI args
cli_args = ["--config", config_file, "--max-running-requests", "128"]
merged_args = merger.merge_config_with_args(cli_args)
# Verify the merged args contain both config and CLI values
assert "--model-path" in merged_args
assert "microsoft/DialoGPT-medium" in merged_args
assert "--host" in merged_args
assert "0.0.0.0" in merged_args
assert "--port" in merged_args
assert "30000" in merged_args
assert "--tensor-parallel-size" in merged_args
assert "2" in merged_args
assert "--max-running-requests" in merged_args
assert "128" in merged_args
# Test boolean arguments
assert "--enable-metrics" in merged_args # True boolean
assert "--stream-output" in merged_args # True boolean
assert "--log-requests" in merged_args # True boolean
assert "--show-time-cost" in merged_args # True boolean
# False booleans should not be present (only add flag if True)
assert "--trust-remote-code" not in merged_args # False boolean
assert "--skip-server-warmup" not in merged_args # False boolean
assert "--is-embedding" not in merged_args # False boolean
finally:
os.unlink(config_file)
def test_server_args_integration():
"""Test the integration with server args."""
# Create a temporary config file
config_data = {
"model-path": "microsoft/DialoGPT-medium",
"host": "0.0.0.0",
"port": 30000,
"tensor-parallel-size": 1,
"max-running-requests": 256,
}
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
yaml.dump(config_data, f)
config_file = f.name
try:
# Test with config file
argv = ["--config", config_file]
server_args = prepare_server_args(argv)
# Verify that config values were loaded
assert server_args.model_path == "microsoft/DialoGPT-medium"
assert server_args.host == "0.0.0.0"
assert server_args.port == 30000
assert server_args.tp_size == 1
assert server_args.max_running_requests == 256
finally:
os.unlink(config_file)
def test_cli_override():
"""Test that CLI arguments override config file values."""
# Create a temporary config file
config_data = {
"model-path": "microsoft/DialoGPT-medium",
"port": 30000,
"tensor-parallel-size": 1,
}
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
yaml.dump(config_data, f)
config_file = f.name
try:
# Test CLI override (CLI should take precedence)
argv = [
"--config",
config_file,
"--port",
"40000",
"--tensor-parallel-size",
"2",
]
server_args = prepare_server_args(argv)
# Verify that CLI values override config values
assert server_args.model_path == "microsoft/DialoGPT-medium" # From config
assert server_args.port == 40000 # From CLI (overrides config)
assert server_args.tp_size == 2 # From CLI (overrides config)
finally:
os.unlink(config_file)
def test_error_handling():
"""Test error handling for invalid config files."""
# Test non-existent config file
with pytest.raises(ValueError, match="Config file not found"):
argv = ["--config", "non-existent.yaml"]
prepare_server_args(argv)
# Test invalid YAML file
with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f:
f.write("invalid: yaml: content: [")
invalid_yaml_file = f.name
try:
with pytest.raises(Exception):
argv = ["--config", invalid_yaml_file]
prepare_server_args(argv)
finally:
os.unlink(invalid_yaml_file)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment