config.go 10.4 KB
Newer Older
1
2
3
package envconfig

import (
4
	"errors"
5
6
	"fmt"
	"log/slog"
7
	"math"
Michael Yang's avatar
Michael Yang committed
8
	"net"
9
10
11
12
13
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
14
	"time"
15
16
)

17
18
19
20
21
22
23
24
25
26
27
28
type OllamaHost struct {
	Scheme string
	Host   string
	Port   string
}

func (o OllamaHost) String() string {
	return fmt.Sprintf("%s://%s:%s", o.Scheme, o.Host, o.Port)
}

var ErrInvalidHostPort = errors.New("invalid port specified in OLLAMA_HOST")

29
30
31
32
33
var (
	// Set via OLLAMA_ORIGINS in the environment
	AllowOrigins []string
	// Set via OLLAMA_DEBUG in the environment
	Debug bool
34
35
	// Experimental flash attention
	FlashAttention bool
36
37
	// Set via OLLAMA_HOST in the environment
	Host *OllamaHost
38
	// Set via OLLAMA_KEEP_ALIVE in the environment
39
	KeepAlive time.Duration
40
41
42
43
44
45
46
47
	// Set via OLLAMA_LLM_LIBRARY in the environment
	LLMLibrary string
	// Set via OLLAMA_MAX_LOADED_MODELS in the environment
	MaxRunners int
	// Set via OLLAMA_MAX_QUEUE in the environment
	MaxQueuedRequests int
	// Set via OLLAMA_MAX_VRAM in the environment
	MaxVRAM uint64
48
49
	// Set via OLLAMA_MODELS in the environment
	ModelsDir string
50
51
	// Set via OLLAMA_NOHISTORY in the environment
	NoHistory bool
52
53
54
55
56
57
	// Set via OLLAMA_NOPRUNE in the environment
	NoPrune bool
	// Set via OLLAMA_NUM_PARALLEL in the environment
	NumParallel int
	// Set via OLLAMA_RUNNERS_DIR in the environment
	RunnersDir string
58
59
	// Set via OLLAMA_SCHED_SPREAD in the environment
	SchedSpread bool
60
61
	// Set via OLLAMA_TMPDIR in the environment
	TmpDir string
62
63
	// Set via OLLAMA_INTEL_GPU in the environment
	IntelGpu bool
64
65
66
67
68
69
70
71
72
73
74

	// Set via CUDA_VISIBLE_DEVICES in the environment
	CudaVisibleDevices string
	// Set via HIP_VISIBLE_DEVICES in the environment
	HipVisibleDevices string
	// Set via ROCR_VISIBLE_DEVICES in the environment
	RocrVisibleDevices string
	// Set via GPU_DEVICE_ORDINAL in the environment
	GpuDeviceOrdinal string
	// Set via HSA_OVERRIDE_GFX_VERSION in the environment
	HsaOverrideGfxVersion string
75
76
)

77
78
79
80
81
82
83
type EnvVar struct {
	Name        string
	Value       any
	Description string
}

func AsMap() map[string]EnvVar {
84
	ret := map[string]EnvVar{
85
86
		"OLLAMA_DEBUG":             {"OLLAMA_DEBUG", Debug, "Show additional debug information (e.g. OLLAMA_DEBUG=1)"},
		"OLLAMA_FLASH_ATTENTION":   {"OLLAMA_FLASH_ATTENTION", FlashAttention, "Enabled flash attention"},
87
		"OLLAMA_HOST":              {"OLLAMA_HOST", Host, "IP Address for the ollama server (default 127.0.0.1:11434)"},
88
		"OLLAMA_KEEP_ALIVE":        {"OLLAMA_KEEP_ALIVE", KeepAlive, "The duration that models stay loaded in memory (default \"5m\")"},
89
		"OLLAMA_LLM_LIBRARY":       {"OLLAMA_LLM_LIBRARY", LLMLibrary, "Set LLM library to bypass autodetection"},
90
		"OLLAMA_MAX_LOADED_MODELS": {"OLLAMA_MAX_LOADED_MODELS", MaxRunners, "Maximum number of loaded models per GPU"},
91
		"OLLAMA_MAX_QUEUE":         {"OLLAMA_MAX_QUEUE", MaxQueuedRequests, "Maximum number of queued requests"},
92
		"OLLAMA_MAX_VRAM":          {"OLLAMA_MAX_VRAM", MaxVRAM, "Maximum VRAM"},
93
		"OLLAMA_MODELS":            {"OLLAMA_MODELS", ModelsDir, "The path to the models directory"},
94
95
		"OLLAMA_NOHISTORY":         {"OLLAMA_NOHISTORY", NoHistory, "Do not preserve readline history"},
		"OLLAMA_NOPRUNE":           {"OLLAMA_NOPRUNE", NoPrune, "Do not prune model blobs on startup"},
96
		"OLLAMA_NUM_PARALLEL":      {"OLLAMA_NUM_PARALLEL", NumParallel, "Maximum number of parallel requests"},
97
		"OLLAMA_ORIGINS":           {"OLLAMA_ORIGINS", AllowOrigins, "A comma separated list of allowed origins"},
98
		"OLLAMA_RUNNERS_DIR":       {"OLLAMA_RUNNERS_DIR", RunnersDir, "Location for runners"},
99
		"OLLAMA_SCHED_SPREAD":      {"OLLAMA_SCHED_SPREAD", SchedSpread, "Always schedule model across all GPUs"},
100
		"OLLAMA_TMPDIR":            {"OLLAMA_TMPDIR", TmpDir, "Location for temporary files"},
101
	}
102
103
104
105
106
107
	if runtime.GOOS != "darwin" {
		ret["CUDA_VISIBLE_DEVICES"] = EnvVar{"CUDA_VISIBLE_DEVICES", CudaVisibleDevices, "Set which NVIDIA devices are visible"}
		ret["HIP_VISIBLE_DEVICES"] = EnvVar{"HIP_VISIBLE_DEVICES", HipVisibleDevices, "Set which AMD devices are visible"}
		ret["ROCR_VISIBLE_DEVICES"] = EnvVar{"ROCR_VISIBLE_DEVICES", RocrVisibleDevices, "Set which AMD devices are visible"}
		ret["GPU_DEVICE_ORDINAL"] = EnvVar{"GPU_DEVICE_ORDINAL", GpuDeviceOrdinal, "Set which AMD devices are visible"}
		ret["HSA_OVERRIDE_GFX_VERSION"] = EnvVar{"HSA_OVERRIDE_GFX_VERSION", HsaOverrideGfxVersion, "Override the gfx used for all detected AMD GPUs"}
108
		ret["OLLAMA_INTEL_GPU"] = EnvVar{"OLLAMA_INTEL_GPU", IntelGpu, "Enable experimental Intel GPU detection"}
109
110
	}
	return ret
111
112
}

113
114
115
116
117
118
119
120
func Values() map[string]string {
	vals := make(map[string]string)
	for k, v := range AsMap() {
		vals[k] = fmt.Sprintf("%v", v.Value)
	}
	return vals
}

121
122
123
124
125
126
127
128
129
130
131
132
133
var defaultAllowOrigins = []string{
	"localhost",
	"127.0.0.1",
	"0.0.0.0",
}

// Clean quotes and spaces from the value
func clean(key string) string {
	return strings.Trim(os.Getenv(key), "\"' ")
}

func init() {
	// default values
134
135
	NumParallel = 0 // Autoselect
	MaxRunners = 0  // Autoselect
136
	MaxQueuedRequests = 512
137
	KeepAlive = 5 * time.Minute
138
139
140
141
142
143
144
145
146
147
148
149
150
151

	LoadConfig()
}

func LoadConfig() {
	if debug := clean("OLLAMA_DEBUG"); debug != "" {
		d, err := strconv.ParseBool(debug)
		if err == nil {
			Debug = d
		} else {
			Debug = true
		}
	}

152
153
154
155
156
157
158
	if fa := clean("OLLAMA_FLASH_ATTENTION"); fa != "" {
		d, err := strconv.ParseBool(fa)
		if err == nil {
			FlashAttention = d
		}
	}

159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
	RunnersDir = clean("OLLAMA_RUNNERS_DIR")
	if runtime.GOOS == "windows" && RunnersDir == "" {
		// On Windows we do not carry the payloads inside the main executable
		appExe, err := os.Executable()
		if err != nil {
			slog.Error("failed to lookup executable path", "error", err)
		}

		cwd, err := os.Getwd()
		if err != nil {
			slog.Error("failed to lookup working directory", "error", err)
		}

		var paths []string
		for _, root := range []string{filepath.Dir(appExe), cwd} {
			paths = append(paths,
Michael Yang's avatar
Michael Yang committed
175
				root,
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
				filepath.Join(root, "windows-"+runtime.GOARCH),
				filepath.Join(root, "dist", "windows-"+runtime.GOARCH),
			)
		}

		// Try a few variations to improve developer experience when building from source in the local tree
		for _, p := range paths {
			candidate := filepath.Join(p, "ollama_runners")
			_, err := os.Stat(candidate)
			if err == nil {
				RunnersDir = candidate
				break
			}
		}
		if RunnersDir == "" {
			slog.Error("unable to locate llm runner directory.  Set OLLAMA_RUNNERS_DIR to the location of 'ollama_runners'")
		}
	}

	TmpDir = clean("OLLAMA_TMPDIR")

	userLimit := clean("OLLAMA_MAX_VRAM")
	if userLimit != "" {
		avail, err := strconv.ParseUint(userLimit, 10, 64)
		if err != nil {
			slog.Error("invalid setting, ignoring", "OLLAMA_MAX_VRAM", userLimit, "error", err)
		} else {
			MaxVRAM = avail
		}
	}

	LLMLibrary = clean("OLLAMA_LLM_LIBRARY")

	if onp := clean("OLLAMA_NUM_PARALLEL"); onp != "" {
		val, err := strconv.Atoi(onp)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
211
212
		if err != nil {
			slog.Error("invalid setting, ignoring", "OLLAMA_NUM_PARALLEL", onp, "error", err)
213
214
215
216
217
		} else {
			NumParallel = val
		}
	}

218
219
220
221
	if nohistory := clean("OLLAMA_NOHISTORY"); nohistory != "" {
		NoHistory = true
	}

222
223
224
225
226
227
228
229
230
	if spread := clean("OLLAMA_SCHED_SPREAD"); spread != "" {
		s, err := strconv.ParseBool(spread)
		if err == nil {
			SchedSpread = s
		} else {
			SchedSpread = true
		}
	}

231
232
233
234
235
236
237
238
239
240
241
	if noprune := clean("OLLAMA_NOPRUNE"); noprune != "" {
		NoPrune = true
	}

	if origins := clean("OLLAMA_ORIGINS"); origins != "" {
		AllowOrigins = strings.Split(origins, ",")
	}
	for _, allowOrigin := range defaultAllowOrigins {
		AllowOrigins = append(AllowOrigins,
			fmt.Sprintf("http://%s", allowOrigin),
			fmt.Sprintf("https://%s", allowOrigin),
Michael Yang's avatar
Michael Yang committed
242
243
			fmt.Sprintf("http://%s", net.JoinHostPort(allowOrigin, "*")),
			fmt.Sprintf("https://%s", net.JoinHostPort(allowOrigin, "*")),
244
245
246
		)
	}

royjhan's avatar
royjhan committed
247
248
249
250
251
252
	AllowOrigins = append(AllowOrigins,
		"app://*",
		"file://*",
		"tauri://*",
	)

253
254
255
256
	maxRunners := clean("OLLAMA_MAX_LOADED_MODELS")
	if maxRunners != "" {
		m, err := strconv.Atoi(maxRunners)
		if err != nil {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
257
			slog.Error("invalid setting, ignoring", "OLLAMA_MAX_LOADED_MODELS", maxRunners, "error", err)
258
259
260
261
262
263
264
265
		} else {
			MaxRunners = m
		}
	}

	if onp := os.Getenv("OLLAMA_MAX_QUEUE"); onp != "" {
		p, err := strconv.Atoi(onp)
		if err != nil || p <= 0 {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
266
			slog.Error("invalid setting, ignoring", "OLLAMA_MAX_QUEUE", onp, "error", err)
267
268
269
270
		} else {
			MaxQueuedRequests = p
		}
	}
271

272
273
274
275
	ka := clean("OLLAMA_KEEP_ALIVE")
	if ka != "" {
		loadKeepAlive(ka)
	}
276
277

	var err error
278
279
280
281
282
	ModelsDir, err = getModelsDir()
	if err != nil {
		slog.Error("invalid setting", "OLLAMA_MODELS", ModelsDir, "error", err)
	}

283
284
285
286
	Host, err = getOllamaHost()
	if err != nil {
		slog.Error("invalid setting", "OLLAMA_HOST", Host, "error", err, "using default port", Host.Port)
	}
287

288
289
290
291
	if set, err := strconv.ParseBool(clean("OLLAMA_INTEL_GPU")); err == nil {
		IntelGpu = set
	}

292
293
294
295
296
	CudaVisibleDevices = clean("CUDA_VISIBLE_DEVICES")
	HipVisibleDevices = clean("HIP_VISIBLE_DEVICES")
	RocrVisibleDevices = clean("ROCR_VISIBLE_DEVICES")
	GpuDeviceOrdinal = clean("GPU_DEVICE_ORDINAL")
	HsaOverrideGfxVersion = clean("HSA_OVERRIDE_GFX_VERSION")
297
298
}

299
300
301
302
303
304
305
306
307
308
309
func getModelsDir() (string, error) {
	if models, exists := os.LookupEnv("OLLAMA_MODELS"); exists {
		return models, nil
	}
	home, err := os.UserHomeDir()
	if err != nil {
		return "", err
	}
	return filepath.Join(home, ".ollama", "models"), nil
}

310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
func getOllamaHost() (*OllamaHost, error) {
	defaultPort := "11434"

	hostVar := os.Getenv("OLLAMA_HOST")
	hostVar = strings.TrimSpace(strings.Trim(strings.TrimSpace(hostVar), "\"'"))

	scheme, hostport, ok := strings.Cut(hostVar, "://")
	switch {
	case !ok:
		scheme, hostport = "http", hostVar
	case scheme == "http":
		defaultPort = "80"
	case scheme == "https":
		defaultPort = "443"
	}

	// trim trailing slashes
	hostport = strings.TrimRight(hostport, "/")

	host, port, err := net.SplitHostPort(hostport)
	if err != nil {
		host, port = "127.0.0.1", defaultPort
		if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
			host = ip.String()
		} else if hostport != "" {
			host = hostport
		}
	}

	if portNum, err := strconv.ParseInt(port, 10, 32); err != nil || portNum > 65535 || portNum < 0 {
		return &OllamaHost{
			Scheme: scheme,
			Host:   host,
			Port:   defaultPort,
		}, ErrInvalidHostPort
	}

	return &OllamaHost{
		Scheme: scheme,
		Host:   host,
		Port:   port,
	}, nil
352
}
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373

func loadKeepAlive(ka string) {
	v, err := strconv.Atoi(ka)
	if err != nil {
		d, err := time.ParseDuration(ka)
		if err == nil {
			if d < 0 {
				KeepAlive = time.Duration(math.MaxInt64)
			} else {
				KeepAlive = d
			}
		}
	} else {
		d := time.Duration(v) * time.Second
		if d < 0 {
			KeepAlive = time.Duration(math.MaxInt64)
		} else {
			KeepAlive = d
		}
	}
}