Skip to content

GitLab

"references/git@developer.sourcefind.cn:OpenDAS/vision.git" did not exist on "1d0786b0a35661408388ed4268e382f56bcde627"
Commit 80163ebc authored by Michael Yang's avatar Michael Yang
Browse files

fix metal gpu

parent a57818d9
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 2 deletions
llm/server.go
View file @ 80163ebc
...@@ -85,7 +85,7 @@ func NewLlamaServer(model string, adapters, projectors []string, opts *api.Optio ...@@ -85,7 +85,7 @@ func NewLlamaServer(model string, adapters, projectors []string, opts *api.Optio
graph := int64(ggml.KV().GQA()) * kv / 6 graph := int64(ggml.KV().GQA()) * kv / 6
usedMemory += graph usedMemory += graph
if usedMemory > availableMemory || slices.Contains(cpuOnlyFamilies, ggml.KV().Architecture()) { if (usedMemory > availableMemory || slices.Contains(cpuOnlyFamilies, ggml.KV().Architecture())) && info.Library != "metal" {
info.Library = "cpu" info.Library = "cpu"
} }
...@@ -159,7 +159,7 @@ func NewLlamaServer(model string, adapters, projectors []string, opts *api.Optio ...@@ -159,7 +159,7 @@ func NewLlamaServer(model string, adapters, projectors []string, opts *api.Optio
params = append(params, "--log-disable") params = append(params, "--log-disable")
} }
if opts.NumGPU > 0 { if opts.NumGPU >= 0 {
params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", opts.NumGPU)) params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", opts.NumGPU))
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment