Commit c5ff443b authored by Daniel Hiltgen's avatar Daniel Hiltgen
Browse files

Handle very slow model loads

During testing, we're seeing some models take over 3 minutes.
parent 1524f323
...@@ -383,7 +383,8 @@ func (s *LlamaServer) Ping(ctx context.Context) error { ...@@ -383,7 +383,8 @@ func (s *LlamaServer) Ping(ctx context.Context) error {
func (s *LlamaServer) waitUntilRunning() error { func (s *LlamaServer) waitUntilRunning() error {
start := time.Now() start := time.Now()
expiresAt := time.Now().Add(3 * time.Minute) // be generous with timeout, large models can take a while to load // TODO we need to wire up a better way to detect hangs during model load and startup of the server
expiresAt := time.Now().Add(10 * time.Minute) // be generous with timeout, large models can take a while to load
ticker := time.NewTicker(50 * time.Millisecond) ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop() defer ticker.Stop()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment