no gpu for 70B model

74a5f7e6 · Michael Yang · 7a1c3e62 · 74a5f7e6
Commit 74a5f7e6 authored Aug 01, 2023 by Michael Yang
Show whitespace changes
Inline Side-by-side

Showing with 5 additions and 0 deletions

llama/llama.go llama/llama.go +5 -0

No files found.
--- a/llama/llama.go
+++ b/llama/llama.go
@@ -128,6 +128,11 @@ func New(model string, opts api.Options) (*LLM, error) {
 	C.llama_backend_init(C.bool(llm.UseNUMA))
+	// TODO: GQA == 8 suggests 70B model which doesn't support metal
+	if llm.NumGQA == 8 {
+		llm.NumGPU = 0
+	}
 	params := C.llama_context_default_params()
 	params.seed = C.uint(llm.Seed)
 	params.n_ctx = C.int(llm.NumCtx)