Load all layers on `arm64` macOS if model is small enough (#2149)

4458efb7 · Jeffrey Morgan · GitHub · ceea5994 · 4458efb7
Unverified Commit 4458efb7 authored Jan 22, 2024 by Jeffrey Morgan Committed by GitHub Jan 22, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 1 deletion

llm/llm.go llm/llm.go +2 -1

No files found.
--- a/llm/llm.go
+++ b/llm/llm.go
@@ -70,7 +70,8 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options)
 			break
 		}

-		opts.NumGPU = 1
+		// TODO: implement layer splitting on macOS
+		opts.NumGPU = 999
 	default:
 		if info.Library == "cpu" {
 			slog.Info("GPU not available, falling back to CPU")