Override numParallel only if unset.

0ad0e738 · Richard Lyons · d29cd4c2 · 0ad0e738
Commit 0ad0e738 authored Aug 18, 2024 by Richard Lyons
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

server/sched.go server/sched.go +3 -1

No files found.
--- a/server/sched.go
+++ b/server/sched.go
@@ -734,7 +734,9 @@ func pickBestFullFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoL
 // If multiple Libraries are detected, pick the Library which loads the most layers for the model
 func pickBestPartialFitByLibrary(req *LlmRequest, ggml *llm.GGML, gpus gpu.GpuInfoList, numParallel *int) gpu.GpuInfoList {
-	*numParallel = 1
+	if *numParallel <= 0 {
+		*numParallel = 1
+        }
 	byLibrary := gpus.ByLibrary()
 	if len(byLibrary) <= 1 {
 		return gpus