Merge pull request #3846 from dhiltgen/missing_runner

Detect and recover if runner removed

.github/workflows/test.yaml

@@ -260,7 +260,7 @@ jobs:
         shell: bash
       - uses: golangci/golangci-lint-action@v4
         with:
-          args: --timeout 8m0s
+          args: --timeout 8m0s -v
   test:
     strategy:
       matrix:

llm/server.go

@@ -250,6 +250,17 @@ func NewLlamaServer(gpus gpu.GpuInfoList, model string, ggml *GGML, adapters, pr
 			server = server + ".exe"
 		}
 
+		// Detect tmp cleaners wiping out the file
+		_, err := os.Stat(server)
+		if errors.Is(err, os.ErrNotExist) {
+			slog.Warn("llama server disappeared, reinitializing payloads", "path", server, "error", err)
+			err = Init()
+			if err != nil {
+				slog.Warn("failed to reinitialize payloads", "error", err)
+				return nil, err
+			}
+		}
+
 		s := &llmServer{
 			port:          port,
 			cmd:           exec.Command(server, finalParams...),