Merge pull request #809 from jmorganca/mxyng/fix-gpu

fix: regression unsupported metal types

Merge pull request #809 from jmorganca/mxyng/fix-gpu
fix: regression unsupported metal types
c031c211 · Michael Yang · GitHub · 68391b00 · cb4a80b6 · c031c211
Unverified Commit c031c211 authored Oct 17, 2023 by Michael Yang Committed by GitHub Oct 17, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 4 deletions

llm/llama.go llm/llama.go +1 -4

No files found.
--- a/llm/llama.go
+++ b/llm/llama.go
@@ -292,13 +292,10 @@ func newLlama(model string, adapters []string, runners []ModelRunner, numLayers
 		"--rope-freq-base", fmt.Sprintf("%f", opts.RopeFrequencyBase),
 		"--rope-freq-scale", fmt.Sprintf("%f", opts.RopeFrequencyScale),
 		"--batch-size", fmt.Sprintf("%d", opts.NumBatch),
+		"--n-gpu-layers", fmt.Sprintf("%d", numGPU),
 		"--embedding",
 	}

-	if numGPU > 0 {
-		params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", numGPU))
-	}
-
 	if opts.NumGQA > 0 {
 		params = append(params, "--gqa", fmt.Sprintf("%d", opts.NumGQA))
 	}