Unverified Commit c2c98725 authored by OlivierDehaene's avatar OlivierDehaene Committed by GitHub
Browse files

fix(router): fix a possible deadlock in next_batch (#1731)

parent 6c2c44b8
...@@ -200,6 +200,10 @@ impl State { ...@@ -200,6 +200,10 @@ impl State {
} }
} }
// Pad prefill_token_budget to be a multiple of block size
let prefill_token_budget =
((prefill_token_budget + self.block_size - 1) / self.block_size) * self.block_size;
// Create span for this batch to add context to inference calls // Create span for this batch to add context to inference calls
let next_batch_span = info_span!(parent: None, "batch", batch_size = tracing::field::Empty); let next_batch_span = info_span!(parent: None, "batch", batch_size = tracing::field::Empty);
next_batch_span.follows_from(&Span::current()); next_batch_span.follows_from(&Span::current());
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment