"llm/llama.cpp/git@developer.sourcefind.cn:OpenDAS/ollama.git" did not exist on "b0c9cd0f3bc6b41b4053fe122192237fa8b68eae"
Unverified Commit da7fac1b authored by Simo Lin's avatar Simo Lin Committed by GitHub
Browse files

[router] allow router launch server to use grpc mode (#11600)

parent 28ad2297
import argparse import argparse
import asyncio
import copy import copy
import logging import logging
import multiprocessing as mp import multiprocessing as mp
...@@ -13,7 +14,6 @@ import requests ...@@ -13,7 +14,6 @@ import requests
from setproctitle import setproctitle from setproctitle import setproctitle
from sglang_router.launch_router import RouterArgs, launch_router from sglang_router.launch_router import RouterArgs, launch_router
from sglang.srt.entrypoints.http_server import launch_server
from sglang.srt.server_args import ServerArgs from sglang.srt.server_args import ServerArgs
from sglang.srt.utils import is_port_available from sglang.srt.utils import is_port_available
...@@ -72,7 +72,15 @@ def run_server(server_args, dp_rank): ...@@ -72,7 +72,15 @@ def run_server(server_args, dp_rank):
# Set SGLANG_DP_RANK environment variable # Set SGLANG_DP_RANK environment variable
os.environ["SGLANG_DP_RANK"] = str(dp_rank) os.environ["SGLANG_DP_RANK"] = str(dp_rank)
launch_server(server_args) # Launch server in appropriate mode (HTTP or gRPC)
if server_args.grpc_mode:
from sglang.srt.entrypoints.grpc_server import serve_grpc
asyncio.run(serve_grpc(server_args))
else:
from sglang.srt.entrypoints.http_server import launch_server
launch_server(server_args)
def launch_server_process( def launch_server_process(
...@@ -186,8 +194,10 @@ def main(): ...@@ -186,8 +194,10 @@ def main():
) )
# Update router args with worker URLs # Update router args with worker URLs
# Use grpc:// protocol if server is in gRPC mode, otherwise http://
protocol = "grpc" if server_args.grpc_mode else "http"
router_args.worker_urls = [ router_args.worker_urls = [
f"http://{server_args.host}:{port}" for port in worker_ports f"{protocol}://{server_args.host}:{port}" for port in worker_ports
] ]
# Start the router # Start the router
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment