Backend method not found when SRT Runtime is used (#1576)

8ac3ccc0 · Byron Hsu · GitHub · 9b0926ce · 8ac3ccc0
Unverified Commit 8ac3ccc0 authored Oct 05, 2024 by Byron Hsu Committed by GitHub Oct 05, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 8 additions and 0 deletions

python/sglang/api.py python/sglang/api.py +8 -0

No files found.
--- a/python/sglang/api.py
+++ b/python/sglang/api.py
@@ -48,6 +48,10 @@ def flush_cache(backend: Optional[BaseBackend] = None):
    backend = backend or global_config.default_backend
    if backend is None:
        return False
+
+    # If backend is Runtime
+    if hasattr(backend, "endpoint"):
+        backend = backend.endpoint
    return backend.flush_cache()


@@ -55,6 +59,10 @@ def get_server_args(backend: Optional[BaseBackend] = None):
    backend = backend or global_config.default_backend
    if backend is None:
        return None
+
+    # If backend is Runtime
+    if hasattr(backend, "endpoint"):
+        backend = backend.endpoint
    return backend.get_server_args()