Merge pull request #1013 from kvcache-ai/work-concurrent

In v0.2.4 version, we’ve added highly desired multi-concurrency support to the community through a major refactor of the whole architecture.

ktransformers/ktransformers_ext/cpu_backend/backend.cpp → csrc/ktransformers_ext/cpu_backend/backend.cpp

@@ -151,4 +151,4 @@ void Backend::worker_thread(int thread_id) {
             return;
         }
     }
-}
+}
\ No newline at end of file

ktransformers/ktransformers_ext/cpu_backend/cpuinfer.h → csrc/ktransformers_ext/cpu_backend/cpuinfer.h

@@ -28,7 +28,7 @@
  
  #include "backend.h"
  #include "task_queue.h"
- #include "../vendors/vendor.h"
+ #include "./vendors/vendor.h"
  
  #include "llama.cpp/ggml-impl.h"
  

ktransformers/ktransformers_ext/cuda/binding.cpp → csrc/ktransformers_ext/cuda/binding.cpp

@@ -68,4 +68,4 @@ PYBIND11_MODULE(KTransformersOps, m) {
         py::arg("perm"), py::arg("workspace"), py::arg("num_bits"), py::arg("size_m"),
         py::arg("size_n"), py::arg("size_k"), py::arg("is_k_full"));
 #endif
-}
+}
\ No newline at end of file

ktransformers/ktransformers_ext/cuda/custom_gguf/dequant.cu → csrc/ktransformers_ext/cuda/custom_gguf/dequant.cu

@@ -879,4 +879,4 @@ torch::Tensor dequantize_iq4_xs(const int8_t* data, const int num_bytes, const i
     }
     cudaDeviceSynchronize();
     return output;
-}
+}
\ No newline at end of file

ktransformers/ktransformers_ext/cuda/custom_gguf/ops.h → csrc/ktransformers_ext/cuda/custom_gguf/ops.h

@@ -19,4 +19,4 @@ torch::Tensor dequantize_q5_k(const int8_t* data, const int num_bytes, const int
 torch::Tensor dequantize_q4_k(const int8_t* data, const int num_bytes, const int blk_size, const int ele_per_blk, const torch::Device device, const torch::Dtype target_dtype);
 torch::Tensor dequantize_q3_k(const int8_t* data, const int num_bytes, const int blk_size, const int ele_per_blk, const torch::Device device, const torch::Dtype target_dtype);
 torch::Tensor dequantize_q2_k(const int8_t* data, const int num_bytes, const int blk_size, const int ele_per_blk, const torch::Device device, const torch::Dtype target_dtype);
-torch::Tensor dequantize_iq4_xs(const int8_t* data, const int num_bytes, const int blk_size, const int ele_per_blk, const torch::Device device, const torch::Dtype target_dtype);
+torch::Tensor dequantize_iq4_xs(const int8_t* data, const int num_bytes, const int blk_size, const int ele_per_blk, const torch::Device device, const torch::Dtype target_dtype);
\ No newline at end of file