"graphbolt/vscode:/vscode.git/clone" did not exist on "29df6ec49ec46a5b04485098134df9869f084711"
Commit ab39e08e authored by Daniel Hiltgen's avatar Daniel Hiltgen Committed by Michael Yang
Browse files

llm: auto detect models that require Ollama Engine (#1)

parent 11bfa627
...@@ -133,6 +133,10 @@ func (kv KV) Floats(key string, defaultValue ...[]float32) []float32 { ...@@ -133,6 +133,10 @@ func (kv KV) Floats(key string, defaultValue ...[]float32) []float32 {
return s return s
} }
func (kv KV) OllamaEngineRequired() bool {
return kv.Architecture() == "gemma3"
}
func keyValue[T string | uint32 | uint64 | float32 | *array | bool](kv KV, key string, defaultValue ...T) T { func keyValue[T string | uint32 | uint64 | float32 | *array | bool](kv KV, key string, defaultValue ...T) T {
if !strings.HasPrefix(key, "tokenizer.") && !strings.HasPrefix(key, "general.") { if !strings.HasPrefix(key, "tokenizer.") && !strings.HasPrefix(key, "general.") {
key = kv.Architecture() + "." + key key = kv.Architecture() + "." + key
......
...@@ -271,7 +271,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, a ...@@ -271,7 +271,7 @@ func NewLlamaServer(gpus discover.GpuInfoList, modelPath string, f *ggml.GGML, a
var llamaModel *llama.Model var llamaModel *llama.Model
var textProcessor model.TextProcessor var textProcessor model.TextProcessor
if envconfig.NewEngine() { if envconfig.NewEngine() || f.KV().OllamaEngineRequired() {
textProcessor, err = model.NewTextProcessor(modelPath) textProcessor, err = model.NewTextProcessor(modelPath)
if err != nil { if err != nil {
// To prepare for opt-out mode, instead of treating this as an error, we fallback to the old runner // To prepare for opt-out mode, instead of treating this as an error, we fallback to the old runner
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment