config.go 10.9 KB
Newer Older
1
2
3
4
5
package envconfig

import (
	"fmt"
	"log/slog"
6
	"math"
Michael Yang's avatar
Michael Yang committed
7
	"net"
Michael Yang's avatar
host  
Michael Yang committed
8
	"net/url"
9
10
11
12
13
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
14
	"time"
15
16
)

Michael Yang's avatar
host  
Michael Yang committed
17
18
19
20
21
// Host returns the scheme and host. Host can be configured via the OLLAMA_HOST environment variable.
// Default is scheme "http" and host "127.0.0.1:11434"
func Host() *url.URL {
	defaultPort := "11434"

Michael Yang's avatar
Michael Yang committed
22
	s := strings.TrimSpace(Var("OLLAMA_HOST"))
Michael Yang's avatar
host  
Michael Yang committed
23
24
25
26
27
28
29
30
31
32
	scheme, hostport, ok := strings.Cut(s, "://")
	switch {
	case !ok:
		scheme, hostport = "http", s
	case scheme == "http":
		defaultPort = "80"
	case scheme == "https":
		defaultPort = "443"
	}

33
	hostport, path, _ := strings.Cut(hostport, "/")
Michael Yang's avatar
host  
Michael Yang committed
34
35
36
37
38
39
40
41
42
43
44
	host, port, err := net.SplitHostPort(hostport)
	if err != nil {
		host, port = "127.0.0.1", defaultPort
		if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
			host = ip.String()
		} else if hostport != "" {
			host = hostport
		}
	}

	if n, err := strconv.ParseInt(port, 10, 32); err != nil || n > 65535 || n < 0 {
Michael Yang's avatar
Michael Yang committed
45
		slog.Warn("invalid port, using default", "port", port, "default", defaultPort)
46
		port = defaultPort
Michael Yang's avatar
host  
Michael Yang committed
47
48
49
50
51
	}

	return &url.URL{
		Scheme: scheme,
		Host:   net.JoinHostPort(host, port),
52
		Path:   path,
Michael Yang's avatar
host  
Michael Yang committed
53
54
55
	}
}

Michael Yang's avatar
origins  
Michael Yang committed
56
57
// Origins returns a list of allowed origins. Origins can be configured via the OLLAMA_ORIGINS environment variable.
func Origins() (origins []string) {
Michael Yang's avatar
Michael Yang committed
58
	if s := Var("OLLAMA_ORIGINS"); s != "" {
Michael Yang's avatar
origins  
Michael Yang committed
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
		origins = strings.Split(s, ",")
	}

	for _, origin := range []string{"localhost", "127.0.0.1", "0.0.0.0"} {
		origins = append(origins,
			fmt.Sprintf("http://%s", origin),
			fmt.Sprintf("https://%s", origin),
			fmt.Sprintf("http://%s", net.JoinHostPort(origin, "*")),
			fmt.Sprintf("https://%s", net.JoinHostPort(origin, "*")),
		)
	}

	origins = append(origins,
		"app://*",
		"file://*",
		"tauri://*",
	)

	return origins
}

Michael Yang's avatar
models  
Michael Yang committed
80
81
82
// Models returns the path to the models directory. Models directory can be configured via the OLLAMA_MODELS environment variable.
// Default is $HOME/.ollama/models
func Models() string {
Michael Yang's avatar
Michael Yang committed
83
	if s := Var("OLLAMA_MODELS"); s != "" {
Michael Yang's avatar
models  
Michael Yang committed
84
85
86
87
88
89
90
91
92
93
94
		return s
	}

	home, err := os.UserHomeDir()
	if err != nil {
		panic(err)
	}

	return filepath.Join(home, ".ollama", "models")
}

Michael Yang's avatar
Michael Yang committed
95
96
97
98
99
// KeepAlive returns the duration that models stay loaded in memory. KeepAlive can be configured via the OLLAMA_KEEP_ALIVE environment variable.
// Negative values are treated as infinite. Zero is treated as no keep alive.
// Default is 5 minutes.
func KeepAlive() (keepAlive time.Duration) {
	keepAlive = 5 * time.Minute
Michael Yang's avatar
Michael Yang committed
100
	if s := Var("OLLAMA_KEEP_ALIVE"); s != "" {
Michael Yang's avatar
Michael Yang committed
101
102
103
104
105
106
107
108
109
110
111
112
113
114
		if d, err := time.ParseDuration(s); err == nil {
			keepAlive = d
		} else if n, err := strconv.ParseInt(s, 10, 64); err == nil {
			keepAlive = time.Duration(n) * time.Second
		}
	}

	if keepAlive < 0 {
		return time.Duration(math.MaxInt64)
	}

	return keepAlive
}

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
// LoadTimeout returns the duration for stall detection during model loads. LoadTimeout can be configured via the OLLAMA_LOAD_TIMEOUT environment variable.
// Zero or Negative values are treated as infinite.
// Default is 5 minutes.
func LoadTimeout() (loadTimeout time.Duration) {
	loadTimeout = 5 * time.Minute
	if s := Var("OLLAMA_LOAD_TIMEOUT"); s != "" {
		if d, err := time.ParseDuration(s); err == nil {
			loadTimeout = d
		} else if n, err := strconv.ParseInt(s, 10, 64); err == nil {
			loadTimeout = time.Duration(n) * time.Second
		}
	}

	if loadTimeout <= 0 {
		return time.Duration(math.MaxInt64)
	}

	return loadTimeout
}

Michael Yang's avatar
bool  
Michael Yang committed
135
136
func Bool(k string) func() bool {
	return func() bool {
Michael Yang's avatar
Michael Yang committed
137
		if s := Var(k); s != "" {
Michael Yang's avatar
bool  
Michael Yang committed
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
			b, err := strconv.ParseBool(s)
			if err != nil {
				return true
			}

			return b
		}

		return false
	}
}

var (
	// Debug enabled additional debug information.
	Debug = Bool("OLLAMA_DEBUG")
	// FlashAttention enables the experimental flash attention feature.
	FlashAttention = Bool("OLLAMA_FLASH_ATTENTION")
	// NoHistory disables readline history.
	NoHistory = Bool("OLLAMA_NOHISTORY")
	// NoPrune disables pruning of model blobs on startup.
	NoPrune = Bool("OLLAMA_NOPRUNE")
	// SchedSpread allows scheduling models across all GPUs.
	SchedSpread = Bool("OLLAMA_SCHED_SPREAD")
	// IntelGPU enables experimental Intel GPU detection.
	IntelGPU = Bool("OLLAMA_INTEL_GPU")
)

Michael Yang's avatar
string  
Michael Yang committed
165
166
func String(s string) func() string {
	return func() string {
Michael Yang's avatar
Michael Yang committed
167
		return Var(s)
Michael Yang's avatar
string  
Michael Yang committed
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
	}
}

var (
	LLMLibrary = String("OLLAMA_LLM_LIBRARY")
	TmpDir     = String("OLLAMA_TMPDIR")

	CudaVisibleDevices    = String("CUDA_VISIBLE_DEVICES")
	HipVisibleDevices     = String("HIP_VISIBLE_DEVICES")
	RocrVisibleDevices    = String("ROCR_VISIBLE_DEVICES")
	GpuDeviceOrdinal      = String("GPU_DEVICE_ORDINAL")
	HsaOverrideGfxVersion = String("HSA_OVERRIDE_GFX_VERSION")
)

func RunnersDir() (p string) {
Michael Yang's avatar
Michael Yang committed
183
	if p := Var("OLLAMA_RUNNERS_DIR"); p != "" {
Michael Yang's avatar
string  
Michael Yang committed
184
185
186
187
188
189
190
191
192
		return p
	}

	if runtime.GOOS != "windows" {
		return
	}

	defer func() {
		if p == "" {
193
			slog.Error("unable to locate llm runner directory. Set OLLAMA_RUNNERS_DIR to the location of 'ollama/runners'")
Michael Yang's avatar
string  
Michael Yang committed
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
		}
	}()

	// On Windows we do not carry the payloads inside the main executable
	exe, err := os.Executable()
	if err != nil {
		return
	}

	cwd, err := os.Getwd()
	if err != nil {
		return
	}

	var paths []string
209
	for _, root := range []string{filepath.Dir(exe), filepath.Join(filepath.Dir(exe), LibRelativeToExe()), cwd} {
Michael Yang's avatar
string  
Michael Yang committed
210
211
		paths = append(paths,
			root,
Daniel Hiltgen's avatar
Daniel Hiltgen committed
212
213
			filepath.Join(root, runtime.GOOS+"-"+runtime.GOARCH),
			filepath.Join(root, "dist", runtime.GOOS+"-"+runtime.GOARCH),
Michael Yang's avatar
string  
Michael Yang committed
214
215
216
217
218
		)
	}

	// Try a few variations to improve developer experience when building from source in the local tree
	for _, path := range paths {
219
		candidate := filepath.Join(path, "lib", "ollama", "runners")
Michael Yang's avatar
string  
Michael Yang committed
220
221
222
223
224
225
226
227
228
		if _, err := os.Stat(candidate); err == nil {
			p = candidate
			break
		}
	}

	return p
}

Michael Yang's avatar
Michael Yang committed
229
230
231
232
233
234
235
func Uint(key string, defaultValue uint) func() uint {
	return func() uint {
		if s := Var(key); s != "" {
			if n, err := strconv.ParseUint(s, 10, 64); err != nil {
				slog.Warn("invalid environment variable, using default", "key", key, "value", s, "default", defaultValue)
			} else {
				return uint(n)
Michael Yang's avatar
int  
Michael Yang committed
236
237
238
			}
		}

Michael Yang's avatar
Michael Yang committed
239
		return defaultValue
Michael Yang's avatar
int  
Michael Yang committed
240
241
242
	}
}

243
var (
Michael Yang's avatar
Michael Yang committed
244
245
246
247
248
249
250
251
	// NumParallel sets the number of parallel model requests. NumParallel can be configured via the OLLAMA_NUM_PARALLEL environment variable.
	NumParallel = Uint("OLLAMA_NUM_PARALLEL", 0)
	// MaxRunners sets the maximum number of loaded models. MaxRunners can be configured via the OLLAMA_MAX_LOADED_MODELS environment variable.
	MaxRunners = Uint("OLLAMA_MAX_LOADED_MODELS", 0)
	// MaxQueue sets the maximum number of queued requests. MaxQueue can be configured via the OLLAMA_MAX_QUEUE environment variable.
	MaxQueue = Uint("OLLAMA_MAX_QUEUE", 512)
	// MaxVRAM sets a maximum VRAM override in bytes. MaxVRAM can be configured via the OLLAMA_MAX_VRAM environment variable.
	MaxVRAM = Uint("OLLAMA_MAX_VRAM", 0)
252
253
)

254
255
256
257
258
259
260
261
262
263
264
265
266
267
func Uint64(key string, defaultValue uint64) func() uint64 {
	return func() uint64 {
		if s := Var(key); s != "" {
			if n, err := strconv.ParseUint(s, 10, 64); err != nil {
				slog.Warn("invalid environment variable, using default", "key", key, "value", s, "default", defaultValue)
			} else {
				return n
			}
		}

		return defaultValue
	}
}

268
269
// Set aside VRAM per GPU
var GpuOverhead = Uint64("OLLAMA_GPU_OVERHEAD", 0)
270

271
272
273
274
275
276
277
type EnvVar struct {
	Name        string
	Value       any
	Description string
}

func AsMap() map[string]EnvVar {
278
	ret := map[string]EnvVar{
Michael Yang's avatar
Michael Yang committed
279
		"OLLAMA_DEBUG":             {"OLLAMA_DEBUG", Debug(), "Show additional debug information (e.g. OLLAMA_DEBUG=1)"},
Michael Yang's avatar
bool  
Michael Yang committed
280
		"OLLAMA_FLASH_ATTENTION":   {"OLLAMA_FLASH_ATTENTION", FlashAttention(), "Enabled flash attention"},
281
		"OLLAMA_GPU_OVERHEAD":      {"OLLAMA_GPU_OVERHEAD", GpuOverhead(), "Reserve a portion of VRAM per GPU (bytes)"},
Michael Yang's avatar
host  
Michael Yang committed
282
		"OLLAMA_HOST":              {"OLLAMA_HOST", Host(), "IP Address for the ollama server (default 127.0.0.1:11434)"},
Michael Yang's avatar
Michael Yang committed
283
		"OLLAMA_KEEP_ALIVE":        {"OLLAMA_KEEP_ALIVE", KeepAlive(), "The duration that models stay loaded in memory (default \"5m\")"},
Michael Yang's avatar
string  
Michael Yang committed
284
		"OLLAMA_LLM_LIBRARY":       {"OLLAMA_LLM_LIBRARY", LLMLibrary(), "Set LLM library to bypass autodetection"},
285
		"OLLAMA_LOAD_TIMEOUT":      {"OLLAMA_LOAD_TIMEOUT", LoadTimeout(), "How long to allow model loads to stall before giving up (default \"5m\")"},
Michael Yang's avatar
int  
Michael Yang committed
286
287
		"OLLAMA_MAX_LOADED_MODELS": {"OLLAMA_MAX_LOADED_MODELS", MaxRunners(), "Maximum number of loaded models per GPU"},
		"OLLAMA_MAX_QUEUE":         {"OLLAMA_MAX_QUEUE", MaxQueue(), "Maximum number of queued requests"},
Michael Yang's avatar
models  
Michael Yang committed
288
		"OLLAMA_MODELS":            {"OLLAMA_MODELS", Models(), "The path to the models directory"},
Michael Yang's avatar
bool  
Michael Yang committed
289
290
		"OLLAMA_NOHISTORY":         {"OLLAMA_NOHISTORY", NoHistory(), "Do not preserve readline history"},
		"OLLAMA_NOPRUNE":           {"OLLAMA_NOPRUNE", NoPrune(), "Do not prune model blobs on startup"},
Michael Yang's avatar
int  
Michael Yang committed
291
		"OLLAMA_NUM_PARALLEL":      {"OLLAMA_NUM_PARALLEL", NumParallel(), "Maximum number of parallel requests"},
Michael Yang's avatar
origins  
Michael Yang committed
292
		"OLLAMA_ORIGINS":           {"OLLAMA_ORIGINS", Origins(), "A comma separated list of allowed origins"},
Michael Yang's avatar
string  
Michael Yang committed
293
		"OLLAMA_RUNNERS_DIR":       {"OLLAMA_RUNNERS_DIR", RunnersDir(), "Location for runners"},
Michael Yang's avatar
bool  
Michael Yang committed
294
		"OLLAMA_SCHED_SPREAD":      {"OLLAMA_SCHED_SPREAD", SchedSpread(), "Always schedule model across all GPUs"},
Michael Yang's avatar
string  
Michael Yang committed
295
		"OLLAMA_TMPDIR":            {"OLLAMA_TMPDIR", TmpDir(), "Location for temporary files"},
Michael Yang's avatar
Michael Yang committed
296
297
298
299
300
301
302
303
304
305
306
307

		// Informational
		"HTTP_PROXY":  {"HTTP_PROXY", String("HTTP_PROXY")(), "HTTP proxy"},
		"HTTPS_PROXY": {"HTTPS_PROXY", String("HTTPS_PROXY")(), "HTTPS proxy"},
		"NO_PROXY":    {"NO_PROXY", String("NO_PROXY")(), "No proxy"},
	}

	if runtime.GOOS != "windows" {
		// Windows environment variables are case-insensitive so there's no need to duplicate them
		ret["http_proxy"] = EnvVar{"http_proxy", String("http_proxy")(), "HTTP proxy"}
		ret["https_proxy"] = EnvVar{"https_proxy", String("https_proxy")(), "HTTPS proxy"}
		ret["no_proxy"] = EnvVar{"no_proxy", String("no_proxy")(), "No proxy"}
308
	}
Michael Yang's avatar
Michael Yang committed
309

310
	if runtime.GOOS != "darwin" {
Michael Yang's avatar
string  
Michael Yang committed
311
312
313
314
315
		ret["CUDA_VISIBLE_DEVICES"] = EnvVar{"CUDA_VISIBLE_DEVICES", CudaVisibleDevices(), "Set which NVIDIA devices are visible"}
		ret["HIP_VISIBLE_DEVICES"] = EnvVar{"HIP_VISIBLE_DEVICES", HipVisibleDevices(), "Set which AMD devices are visible"}
		ret["ROCR_VISIBLE_DEVICES"] = EnvVar{"ROCR_VISIBLE_DEVICES", RocrVisibleDevices(), "Set which AMD devices are visible"}
		ret["GPU_DEVICE_ORDINAL"] = EnvVar{"GPU_DEVICE_ORDINAL", GpuDeviceOrdinal(), "Set which AMD devices are visible"}
		ret["HSA_OVERRIDE_GFX_VERSION"] = EnvVar{"HSA_OVERRIDE_GFX_VERSION", HsaOverrideGfxVersion(), "Override the gfx used for all detected AMD GPUs"}
Michael Yang's avatar
bool  
Michael Yang committed
316
		ret["OLLAMA_INTEL_GPU"] = EnvVar{"OLLAMA_INTEL_GPU", IntelGPU(), "Enable experimental Intel GPU detection"}
317
	}
Michael Yang's avatar
Michael Yang committed
318

319
	return ret
320
321
}

322
323
324
325
326
327
328
329
func Values() map[string]string {
	vals := make(map[string]string)
	for k, v := range AsMap() {
		vals[k] = fmt.Sprintf("%v", v.Value)
	}
	return vals
}

Michael Yang's avatar
Michael Yang committed
330
331
332
// Var returns an environment variable stripped of leading and trailing quotes or spaces
func Var(key string) string {
	return strings.Trim(strings.TrimSpace(os.Getenv(key)), "\"'")
333
}
334
335
336
337
338
339
340
341
342

// On windows, we keep the binary at the top directory, but
// other platforms use a "bin" directory, so this returns ".."
func LibRelativeToExe() string {
	if runtime.GOOS == "windows" {
		return "."
	}
	return ".."
}