fix(router): Obey max batch size (#23)

f7ac3949 · Nick Hill · GitHub · e6d3eb5d · f7ac3949
Unverified Commit f7ac3949 authored Jan 17, 2023 by Nick Hill Committed by GitHub Jan 17, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

router/src/batcher.rs router/src/batcher.rs +1 -1

No files found.
--- a/router/src/batcher.rs
+++ b/router/src/batcher.rs
@@ -127,7 +127,7 @@ async fn batching_task(
                    // Try to get a new batch
                    if let Some((new_request_ids, new_batch)) =
-                        db.next_batch(min_size, max_batch_size)
+                        db.next_batch(min_size, max_batch_size - batch_size as usize)
                    {
                        // Generate one token for this new batch to have the attention past in cache
                        let new_cached_batch =