add balance-serve, support concurrence

25cee581 · Atream · 8d0292aa · 25cee581 · 25cee581 · 25cee581
Commit 25cee581 authored Mar 31, 2025 by Atream
20 changed files
--- a/ktransformers/operators/RoPE.py
+++ b/ktransformers/operators/RoPE.py
--- a/ktransformers/operators/attention.py
+++ b/ktransformers/operators/attention.py
--- a/ktransformers/operators/experts.py
+++ b/ktransformers/operators/experts.py
--- a/ktransformers/operators/flashinfer_wrapper.py
+++ b/ktransformers/operators/flashinfer_wrapper.py
--- a/ktransformers/operators/gate.py
+++ b/ktransformers/operators/gate.py
--- a/ktransformers/operators/layernorm.py
+++ b/ktransformers/operators/layernorm.py
--- a/ktransformers/operators/linear.py
+++ b/ktransformers/operators/linear.py
--- a/ktransformers/operators/mlp.py
+++ b/ktransformers/operators/mlp.py
--- a/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat.yaml
+++ b/ktransformers/optimize/optimize_rules/DeepSeek-V2-Lite-Chat.yaml
--- a/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-fp8-linear-ggml-experts-serve.yaml
+++ b/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-fp8-linear-ggml-experts-serve.yaml
--- a/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-multi-gpu-fp8-linear-ggml-experts.yaml
+++ b/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-multi-gpu-fp8-linear-ggml-experts.yaml
--- a/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-multi-gpu-marlin.yaml
+++ b/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-multi-gpu-marlin.yaml
--- a/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-serve.yaml
+++ b/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat-serve.yaml
--- a/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat.yaml
+++ b/ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat.yaml
--- a/ktransformers/optimize/optimize_rules/Moonlight-16B-A3B-serve.yaml
+++ b/ktransformers/optimize/optimize_rules/Moonlight-16B-A3B-serve.yaml
--- a/ktransformers/optimize/optimize_rules/Moonlight-16B-A3B.yaml
+++ b/ktransformers/optimize/optimize_rules/Moonlight-16B-A3B.yaml
--- a/ktransformers/server/args.py
+++ b/ktransformers/server/args.py
--- a/ktransformers/server/backend/args.py
+++ b/ktransformers/server/backend/args.py
--- a/ktransformers/server/backend/context_manager.py
+++ b/ktransformers/server/backend/context_manager.py
--- a/ktransformers/server/backend/interfaces/balance_serve.py
+++ b/ktransformers/server/backend/interfaces/balance_serve.py