Skip to content

GitLab

Commit 36fe2dee authored by Michael Yang's avatar Michael Yang
Browse files

only check system memory on macos

parent 4a8931f6
Show whitespace changes
Inline Side-by-side
Showing with 21 additions and 28 deletions
llm/llm.go
View file @ 36fe2dee
...@@ -56,7 +56,6 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error ...@@ -56,7 +56,6 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error
opts.NumGPU = 0 opts.NumGPU = 0
} }
} }
}
var requiredMemory int64 var requiredMemory int64
var f16Multiplier int64 = 2 var f16Multiplier int64 = 2
...@@ -77,18 +76,12 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error ...@@ -77,18 +76,12 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error
systemMemory := int64(memory.TotalMemory()) systemMemory := int64(memory.TotalMemory())
videoMemory, err := CheckVRAM() if ggml.FileType() == "F16" && requiredMemory*f16Multiplier > systemMemory {
if err != nil{
videoMemory = 0
}
totalMemory := systemMemory + videoMemory
if ggml.FileType() == "F16" && requiredMemory*f16Multiplier > totalMemory {
return nil, fmt.Errorf("F16 model requires at least %s of total memory", format.HumanBytes(requiredMemory)) return nil, fmt.Errorf("F16 model requires at least %s of total memory", format.HumanBytes(requiredMemory))
} else if requiredMemory > totalMemory { } else if requiredMemory > systemMemory {
return nil, fmt.Errorf("model requires at least %s of total memory", format.HumanBytes(requiredMemory)) return nil, fmt.Errorf("model requires at least %s of total memory", format.HumanBytes(requiredMemory))
} }
}
switch ggml.Name() { switch ggml.Name() {
case "gguf": case "gguf":
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment