fix(server): revert gpt-neox optims (#123)

cbd36aa4 · OlivierDehaene · GitHub · 6860ce9c · cbd36aa4 · cbd36aa4
Unverified Commit cbd36aa4 authored Mar 13, 2023 by OlivierDehaene Committed by GitHub Mar 13, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 6 additions and 2 deletions

router/src/infer.rs router/src/infer.rs +5 -1

server/Makefile server/Makefile +1 -1

No files found.
--- a/router/src/infer.rs
+++ b/router/src/infer.rs
@@ -229,7 +229,11 @@ async fn batching_task(
    shared: Arc<Shared>,
 ) {
    // Minimum batch size after which we try to add more requests
-    let limit_min_batch_size = (max_batch_size / 2) as u32;
+    let limit_min_batch_size = if max_batch_size > 1 {
+        (max_batch_size / 2) as u32
+    } else {
+        0
+    };

    // Infinite loop
    loop {

--- a/server/Makefile
+++ b/server/Makefile
-transformers_commit := 2f87dca1ca3e5663d0637da9bb037a6956e57a5e
+transformers_commit := 517563354a3226ecfc3dca6e7a38012668d7156a

 gen-server:
 	# Compile protos