add balance-serve, support concurrence

25cee581 · Atream · 8d0292aa · 25cee581 · 25cee581 · 25cee581
Commit 25cee581 authored Mar 31, 2025 by Atream
20 changed files
--- a/ktransformers/server/backend/interfaces/ktransformers.py
+++ b/ktransformers/server/backend/interfaces/ktransformers.py
--- a/ktransformers/server/balance_serve/inference/__init__.py
+++ b/ktransformers/server/balance_serve/inference/__init__.py
--- a/ktransformers/server/balance_serve/inference/config.py
+++ b/ktransformers/server/balance_serve/inference/config.py
--- a/ktransformers/server/balance_serve/inference/distributed/__init__.py
+++ b/ktransformers/server/balance_serve/inference/distributed/__init__.py
--- a/ktransformers/server/balance_serve/inference/distributed/communication_op.py
+++ b/ktransformers/server/balance_serve/inference/distributed/communication_op.py
--- a/ktransformers/server/balance_serve/inference/distributed/cuda_wrapper.py
+++ b/ktransformers/server/balance_serve/inference/distributed/cuda_wrapper.py
--- a/ktransformers/server/balance_serve/inference/distributed/custom_all_reduce.py
+++ b/ktransformers/server/balance_serve/inference/distributed/custom_all_reduce.py
--- a/ktransformers/server/balance_serve/inference/distributed/custom_all_reduce_utils.py
+++ b/ktransformers/server/balance_serve/inference/distributed/custom_all_reduce_utils.py
--- a/ktransformers/server/balance_serve/inference/distributed/parallel_state.py
+++ b/ktransformers/server/balance_serve/inference/distributed/parallel_state.py
--- a/ktransformers/server/balance_serve/inference/distributed/pynccl.py
+++ b/ktransformers/server/balance_serve/inference/distributed/pynccl.py
--- a/ktransformers/server/balance_serve/inference/distributed/pynccl_wrapper.py
+++ b/ktransformers/server/balance_serve/inference/distributed/pynccl_wrapper.py
--- a/ktransformers/server/balance_serve/inference/distributed/utils.py
+++ b/ktransformers/server/balance_serve/inference/distributed/utils.py
--- a/ktransformers/server/balance_serve/inference/forward_batch.py
+++ b/ktransformers/server/balance_serve/inference/forward_batch.py
--- a/ktransformers/server/balance_serve/inference/model_runner.py
+++ b/ktransformers/server/balance_serve/inference/model_runner.py
--- a/ktransformers/server/balance_serve/inference/query_manager.py
+++ b/ktransformers/server/balance_serve/inference/query_manager.py
--- a/ktransformers/server/balance_serve/inference/sampling/penaltylib/__init__.py
+++ b/ktransformers/server/balance_serve/inference/sampling/penaltylib/__init__.py
--- a/ktransformers/server/balance_serve/inference/sampling/penaltylib/orchestrator.py
+++ b/ktransformers/server/balance_serve/inference/sampling/penaltylib/orchestrator.py
--- a/ktransformers/server/balance_serve/inference/sampling/penaltylib/penalizers/frequency_penalty.py
+++ b/ktransformers/server/balance_serve/inference/sampling/penaltylib/penalizers/frequency_penalty.py
--- a/ktransformers/server/balance_serve/inference/sampling/penaltylib/penalizers/min_new_tokens.py
+++ b/ktransformers/server/balance_serve/inference/sampling/penaltylib/penalizers/min_new_tokens.py
--- a/ktransformers/server/balance_serve/inference/sampling/penaltylib/penalizers/presence_penalty.py
+++ b/ktransformers/server/balance_serve/inference/sampling/penaltylib/penalizers/presence_penalty.py