Skip to content

GitLab

Unverified Commit 3144e2a4 authored by Bruce MacDonald's avatar Bruce MacDonald Committed by GitHub
Browse files

exponential back-off (#1484)

parent c0960e29
Show whitespace changes
Inline Side-by-side
Showing with 3 additions and 2 deletions
llm/llama.go
View file @ 3144e2a4
...@@ -545,8 +545,7 @@ type prediction struct { ...@@ -545,8 +545,7 @@ type prediction struct {
} }
const maxBufferSize = 512 * format.KiloByte const maxBufferSize = 512 * format.KiloByte
const maxRetries = 3 const maxRetries = 6
const retryDelay = 1 * time.Second
type PredictOpts struct { type PredictOpts struct {
Prompt string Prompt string
...@@ -610,9 +609,11 @@ func (llm *llama) Predict(ctx context.Context, predict PredictOpts, fn func(Pred ...@@ -610,9 +609,11 @@ func (llm *llama) Predict(ctx context.Context, predict PredictOpts, fn func(Pred
request["grammar"] = jsonGrammar request["grammar"] = jsonGrammar
} }
retryDelay := 100 * time.Microsecond
for retries := 0; retries < maxRetries; retries++ { for retries := 0; retries < maxRetries; retries++ {
if retries > 0 { if retries > 0 {
time.Sleep(retryDelay) // wait before retrying time.Sleep(retryDelay) // wait before retrying
retryDelay *= 2 // exponential backoff
} }
// Handling JSON marshaling with special characters unescaped. // Handling JSON marshaling with special characters unescaped.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment