"tools/run_api_server.py" did not exist on "fffa049707b7534595acd009246827f3a58110b1"
llm.go 2.5 KB
Newer Older
1
2
3
package llm

import (
4
	"context"
5
	"fmt"
6
	"log"
7
	"os"
8
	"runtime"
9

10
11
	"github.com/pbnjay/memory"

12
	"github.com/jmorganca/ollama/api"
Michael Yang's avatar
Michael Yang committed
13
	"github.com/jmorganca/ollama/format"
14
	"github.com/jmorganca/ollama/gpu"
15
16
17
)

type LLM interface {
Bruce MacDonald's avatar
Bruce MacDonald committed
18
	Predict(context.Context, PredictOpts, func(PredictResult)) error
19
20
21
	Embedding(context.Context, string) ([]float64, error)
	Encode(context.Context, string) ([]int, error)
	Decode(context.Context, []int) (string, error)
22
23
24
	Close()
}

25
var AvailableShims = map[string]string{}
26

Michael Yang's avatar
Michael Yang committed
27
func New(workDir, model string, adapters, projectors []string, opts api.Options) (LLM, error) {
28
29
30
31
32
33
34
35
	if _, err := os.Stat(model); err != nil {
		return nil, err
	}

	f, err := os.Open(model)
	if err != nil {
		return nil, err
	}
Michael Yang's avatar
Michael Yang committed
36
	defer f.Close()
37

Bruce MacDonald's avatar
Bruce MacDonald committed
38
	ggml, err := DecodeGGML(f)
39
40
41
42
	if err != nil {
		return nil, err
	}

43
	if runtime.GOOS == "darwin" {
44
45
		var requiredMemory int64
		var f16Multiplier int64 = 2
46

47
48
49
50
51
52
53
		switch ggml.ModelType() {
		case "3B", "7B":
			requiredMemory = 8 * format.GigaByte
		case "13B":
			requiredMemory = 16 * format.GigaByte
		case "30B", "34B", "40B":
			requiredMemory = 32 * format.GigaByte
54
55
		case "47B":
			requiredMemory = 48 * format.GigaByte
56
57
58
59
60
61
		case "65B", "70B":
			requiredMemory = 64 * format.GigaByte
		case "180B":
			requiredMemory = 128 * format.GigaByte
			f16Multiplier = 4
		}
62

63
		systemMemory := int64(memory.TotalMemory())
64

65
66
67
68
69
		if ggml.FileType() == "F16" && requiredMemory*f16Multiplier > systemMemory {
			return nil, fmt.Errorf("F16 model requires at least %s of total memory", format.HumanBytes(requiredMemory))
		} else if requiredMemory > systemMemory {
			return nil, fmt.Errorf("model requires at least %s of total memory", format.HumanBytes(requiredMemory))
		}
70
71
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
72
73
74
	opts.NumGQA = 0
	opts.RopeFrequencyBase = 0.0
	opts.RopeFrequencyScale = 0.0
75
	gpuInfo := gpu.GetGPUInfo()
76
	return newLlmServer(gpuInfo.Library, model, adapters, projectors, ggml.NumLayers(), opts)
77
78
79
80
81
}

// Give any native cgo implementations an opportunity to initialize
func Init(workdir string) error {
	return nativeInit(workdir)
82
}
83
84
85
86
87
88
89
90
91
92
93
94
95

func newLlmServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
	if _, libPresent := AvailableShims[library]; libPresent && library != "default" {
		srv, err := newDynamicShimExtServer(AvailableShims[library], model, adapters, projectors, numLayers, opts)
		if err == nil {
			return srv, nil
		}
		log.Printf("Failed to load dynamic library - falling back to CPU mode %s", err)
	}

	return newDefaultExtServer(model, adapters, projectors, numLayers, opts)

}