add balance-serve, support concurrence

25cee581 · Atream · 8d0292aa · 25cee581 · 25cee581 · 25cee581
Commit 25cee581 authored Mar 31, 2025 by Atream
20 changed files
--- a/csrc/custom_marlin/__init__.py
+++ b/csrc/custom_marlin/__init__.py
--- a/csrc/custom_marlin/binding.cpp
+++ b/csrc/custom_marlin/binding.cpp
--- a/csrc/custom_marlin/gptq_marlin/gptq_marlin.cu
+++ b/csrc/custom_marlin/gptq_marlin/gptq_marlin.cu
--- a/csrc/custom_marlin/gptq_marlin/gptq_marlin.cuh
+++ b/csrc/custom_marlin/gptq_marlin/gptq_marlin.cuh
--- a/csrc/custom_marlin/gptq_marlin/gptq_marlin_dtypes.cuh
+++ b/csrc/custom_marlin/gptq_marlin/gptq_marlin_dtypes.cuh
--- a/csrc/custom_marlin/gptq_marlin/gptq_marlin_repack.cu
+++ b/csrc/custom_marlin/gptq_marlin/gptq_marlin_repack.cu
--- a/csrc/custom_marlin/gptq_marlin/ops.h
+++ b/csrc/custom_marlin/gptq_marlin/ops.h
--- a/csrc/custom_marlin/setup.py
+++ b/csrc/custom_marlin/setup.py
--- a/csrc/custom_marlin/test_cuda_graph.py
+++ b/csrc/custom_marlin/test_cuda_graph.py
--- a/csrc/custom_marlin/utils/__init__.py
+++ b/csrc/custom_marlin/utils/__init__.py
--- a/csrc/custom_marlin/utils/format24.py
+++ b/csrc/custom_marlin/utils/format24.py
--- a/csrc/custom_marlin/utils/marlin_24_perms.py
+++ b/csrc/custom_marlin/utils/marlin_24_perms.py
--- a/csrc/custom_marlin/utils/marlin_perms.py
+++ b/csrc/custom_marlin/utils/marlin_perms.py
--- a/csrc/custom_marlin/utils/marlin_utils.py
+++ b/csrc/custom_marlin/utils/marlin_utils.py
--- a/csrc/custom_marlin/utils/quant_utils.py
+++ b/csrc/custom_marlin/utils/quant_utils.py
--- a/csrc/ktransformers_ext/CMakeLists.txt
+++ b/csrc/ktransformers_ext/CMakeLists.txt
--- a/csrc/ktransformers_ext/cpu_backend/backend.cpp
+++ b/csrc/ktransformers_ext/cpu_backend/backend.cpp
--- a/csrc/ktransformers_ext/cpu_backend/cpuinfer.h
+++ b/csrc/ktransformers_ext/cpu_backend/cpuinfer.h
--- a/csrc/ktransformers_ext/cuda/binding.cpp
+++ b/csrc/ktransformers_ext/cuda/binding.cpp
--- a/csrc/ktransformers_ext/cuda/custom_gguf/dequant.cu
+++ b/csrc/ktransformers_ext/cuda/custom_gguf/dequant.cu