gpu.go 22.2 KB
Newer Older
1
2
3
4
5
//go:build linux || windows

package gpu

/*
6
7
8
#cgo linux LDFLAGS: -lrt -lpthread -ldl -lstdc++ -lm
#cgo windows LDFLAGS: -lpthread

9
10
11
#include "gpu_info.h"
*/
import "C"
Michael Yang's avatar
lint  
Michael Yang committed
12

13
14
import (
	"fmt"
15
	"log/slog"
16
17
	"os"
	"path/filepath"
18
	"runtime"
19
	"strings"
20
21
	"sync"
	"unsafe"
Michael Yang's avatar
Michael Yang committed
22

23
	"github.com/ollama/ollama/envconfig"
24
	"github.com/ollama/ollama/format"
25
26
)

Daniel Hiltgen's avatar
Daniel Hiltgen committed
27
type cudaHandles struct {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
28
29
	deviceCount int
	cudart      *C.cudart_handle_t
30
	nvcuda      *C.nvcuda_handle_t
31
	nvml        *C.nvml_handle_t
Daniel Hiltgen's avatar
Daniel Hiltgen committed
32
33
34
}

type oneapiHandles struct {
Wang,Zhe's avatar
Wang,Zhe committed
35
	oneapi      *C.oneapi_handle_t
Daniel Hiltgen's avatar
Daniel Hiltgen committed
36
	deviceCount int
37
38
}

Michael Yang's avatar
Michael Yang committed
39
const (
Daniel Hiltgen's avatar
Daniel Hiltgen committed
40
41
	cudaMinimumMemory = 457 * format.MebiByte
	rocmMinimumMemory = 457 * format.MebiByte
Daniel Hiltgen's avatar
Daniel Hiltgen committed
42
	// TODO OneAPI minimum memory
Michael Yang's avatar
Michael Yang committed
43
44
)

45
46
47
48
49
50
51
52
53
var (
	gpuMutex      sync.Mutex
	bootstrapped  bool
	cpuCapability CPUCapability
	cpus          []CPUInfo
	cudaGPUs      []CudaGPUInfo
	nvcudaLibPath string
	cudartLibPath string
	oneapiLibPath string
54
	nvmlLibPath   string
55
56
	rocmGPUs      []RocmGPUInfo
	oneapiGPUs    []OneapiGPUInfo
57
58
59
60
61
62
63

	// If any discovered GPUs are incompatible, report why
	unsupportedGPUs []UnsupportedGPUInfo

	// Keep track of errors during bootstrapping so that if GPUs are missing
	// they expected to be present this may explain why
	bootstrapErrors []error
64
)
65

66
67
// With our current CUDA compile flags, older than 5.0 will not work properly
var CudaComputeMin = [2]C.int{5, 0}
68

Daniel Hiltgen's avatar
Daniel Hiltgen committed
69
var RocmComputeMin = 9
70

Daniel Hiltgen's avatar
Daniel Hiltgen committed
71
72
// TODO find a better way to detect iGPU instead of minimum memory
const IGPUMemLimit = 1 * format.GibiByte // 512G is what they typically report, so anything less than 1G must be iGPU
73

74
// Note: gpuMutex must already be held
Daniel Hiltgen's avatar
Daniel Hiltgen committed
75
func initCudaHandles() *cudaHandles {
76
	// TODO - if the ollama build is CPU only, don't do these checks as they're irrelevant and confusing
77

Daniel Hiltgen's avatar
Daniel Hiltgen committed
78
	cHandles := &cudaHandles{}
79
	// Short Circuit if we already know which library to use
80
	// ignore bootstrap errors in this case since we already recorded them
81
	if nvmlLibPath != "" {
82
		cHandles.nvml, _, _ = loadNVMLMgmt([]string{nvmlLibPath})
83
84
		return cHandles
	}
85
	if nvcudaLibPath != "" {
86
		cHandles.deviceCount, cHandles.nvcuda, _, _ = loadNVCUDAMgmt([]string{nvcudaLibPath})
Daniel Hiltgen's avatar
Daniel Hiltgen committed
87
		return cHandles
88
89
	}
	if cudartLibPath != "" {
90
		cHandles.deviceCount, cHandles.cudart, _, _ = loadCUDARTMgmt([]string{cudartLibPath})
Daniel Hiltgen's avatar
Daniel Hiltgen committed
91
		return cHandles
92
93
94
	}

	slog.Debug("searching for GPU discovery libraries for NVIDIA")
95
	var cudartMgmtPatterns []string
96

Daniel Hiltgen's avatar
Daniel Hiltgen committed
97
98
	// Aligned with driver, we can't carry as payloads
	nvcudaMgmtPatterns := NvcudaGlobs
99

Daniel Hiltgen's avatar
Daniel Hiltgen committed
100
101
102
103
	if runtime.GOOS == "windows" {
		localAppData := os.Getenv("LOCALAPPDATA")
		cudartMgmtPatterns = []string{filepath.Join(localAppData, "Programs", "Ollama", CudartMgmtName)}
	}
104
105
106
	libDir := LibraryDir()
	if libDir != "" {
		cudartMgmtPatterns = []string{filepath.Join(libDir, CudartMgmtName)}
107
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
108
	cudartMgmtPatterns = append(cudartMgmtPatterns, CudartGlobs...)
109

Daniel Hiltgen's avatar
Daniel Hiltgen committed
110
111
	if len(NvmlGlobs) > 0 {
		nvmlLibPaths := FindGPULibs(NvmlMgmtName, NvmlGlobs)
112
		if len(nvmlLibPaths) > 0 {
113
			nvml, libPath, err := loadNVMLMgmt(nvmlLibPaths)
114
115
116
117
118
			if nvml != nil {
				slog.Debug("nvidia-ml loaded", "library", libPath)
				cHandles.nvml = nvml
				nvmlLibPath = libPath
			}
119
120
121
			if err != nil {
				bootstrapErrors = append(bootstrapErrors, err)
			}
122
123
124
		}
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
125
	nvcudaLibPaths := FindGPULibs(NvcudaMgmtName, nvcudaMgmtPatterns)
126
	if len(nvcudaLibPaths) > 0 {
127
		deviceCount, nvcuda, libPath, err := loadNVCUDAMgmt(nvcudaLibPaths)
128
		if nvcuda != nil {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
129
			slog.Debug("detected GPUs", "count", deviceCount, "library", libPath)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
130
131
			cHandles.nvcuda = nvcuda
			cHandles.deviceCount = deviceCount
132
			nvcudaLibPath = libPath
Daniel Hiltgen's avatar
Daniel Hiltgen committed
133
			return cHandles
134
		}
135
136
137
		if err != nil {
			bootstrapErrors = append(bootstrapErrors, err)
		}
138
139
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
140
	cudartLibPaths := FindGPULibs(CudartMgmtName, cudartMgmtPatterns)
141
	if len(cudartLibPaths) > 0 {
142
		deviceCount, cudart, libPath, err := loadCUDARTMgmt(cudartLibPaths)
143
		if cudart != nil {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
144
			slog.Debug("detected GPUs", "library", libPath, "count", deviceCount)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
145
146
			cHandles.cudart = cudart
			cHandles.deviceCount = deviceCount
147
			cudartLibPath = libPath
Daniel Hiltgen's avatar
Daniel Hiltgen committed
148
			return cHandles
149
		}
150
151
152
		if err != nil {
			bootstrapErrors = append(bootstrapErrors, err)
		}
153
	}
Wang,Zhe's avatar
Wang,Zhe committed
154

Daniel Hiltgen's avatar
Daniel Hiltgen committed
155
156
157
158
159
160
161
162
	return cHandles
}

// Note: gpuMutex must already be held
func initOneAPIHandles() *oneapiHandles {
	oHandles := &oneapiHandles{}

	// Short Circuit if we already know which library to use
163
	// ignore bootstrap errors in this case since we already recorded them
Daniel Hiltgen's avatar
Daniel Hiltgen committed
164
	if oneapiLibPath != "" {
165
		oHandles.deviceCount, oHandles.oneapi, _, _ = loadOneapiMgmt([]string{oneapiLibPath})
Daniel Hiltgen's avatar
Daniel Hiltgen committed
166
167
168
		return oHandles
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
169
	oneapiLibPaths := FindGPULibs(OneapiMgmtName, OneapiGlobs)
170
	if len(oneapiLibPaths) > 0 {
171
172
173
174
175
		var err error
		oHandles.deviceCount, oHandles.oneapi, oneapiLibPath, err = loadOneapiMgmt(oneapiLibPaths)
		if err != nil {
			bootstrapErrors = append(bootstrapErrors, err)
		}
176
177
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
178
	return oHandles
179
180
}

181
182
183
184
185
186
187
188
189
190
191
func GetCPUInfo() GpuInfoList {
	gpuMutex.Lock()
	if !bootstrapped {
		gpuMutex.Unlock()
		GetGPUInfo()
	} else {
		gpuMutex.Unlock()
	}
	return GpuInfoList{cpus[0].GpuInfo}
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
192
func GetGPUInfo() GpuInfoList {
193
194
195
196
	// TODO - consider exploring lspci (and equivalent on windows) to check for
	// GPUs so we can report warnings if we see Nvidia/AMD but fail to load the libraries
	gpuMutex.Lock()
	defer gpuMutex.Unlock()
197
	needRefresh := true
Daniel Hiltgen's avatar
Daniel Hiltgen committed
198
199
	var cHandles *cudaHandles
	var oHandles *oneapiHandles
200
	defer func() {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
201
202
203
204
205
206
207
		if cHandles != nil {
			if cHandles.cudart != nil {
				C.cudart_release(*cHandles.cudart)
			}
			if cHandles.nvcuda != nil {
				C.nvcuda_release(*cHandles.nvcuda)
			}
208
209
210
			if cHandles.nvml != nil {
				C.nvml_release(*cHandles.nvml)
			}
211
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
212
213
214
215
216
		if oHandles != nil {
			if oHandles.oneapi != nil {
				// TODO - is this needed?
				C.oneapi_release(*oHandles.oneapi)
			}
217
		}
218
	}()
219

220
	if !bootstrapped {
221
		slog.Info("looking for compatible GPUs")
222
		bootstrapErrors = []error{}
223
		needRefresh = false
Daniel Hiltgen's avatar
Daniel Hiltgen committed
224
		cpuCapability = GetCPUCapability()
225
		var memInfo C.mem_info_t
226
227
228
229

		mem, err := GetCPUMem()
		if err != nil {
			slog.Warn("error looking up system memory", "error", err)
230
		}
231
232
		depPath := LibraryDir()

Michael Yang's avatar
lint  
Michael Yang committed
233
234
235
		cpus = []CPUInfo{
			{
				GpuInfo: GpuInfo{
236
237
238
239
240
					memInfo:        mem,
					Library:        "cpu",
					Variant:        cpuCapability.String(),
					ID:             "0",
					DependencyPath: depPath,
Michael Yang's avatar
lint  
Michael Yang committed
241
				},
242
			},
Michael Yang's avatar
lint  
Michael Yang committed
243
		}
244
245
246

		// Fallback to CPU mode if we're lacking required vector extensions on x86
		if cpuCapability < GPURunnerCPUCapability && runtime.GOARCH == "amd64" {
247
248
249
			err := fmt.Errorf("CPU does not have minimum vector extensions, GPU inference disabled.  Required:%s  Detected:%s", GPURunnerCPUCapability, cpuCapability)
			slog.Warn(err.Error())
			bootstrapErrors = append(bootstrapErrors, err)
250
251
252
253
			bootstrapped = true
			// No need to do any GPU discovery, since we can't run on them
			return GpuInfoList{cpus[0].GpuInfo}
		}
254

255
		// Load ALL libraries
Daniel Hiltgen's avatar
Daniel Hiltgen committed
256
		cHandles = initCudaHandles()
257
258

		// NVIDIA
Daniel Hiltgen's avatar
Daniel Hiltgen committed
259
260
		for i := range cHandles.deviceCount {
			if cHandles.cudart != nil || cHandles.nvcuda != nil {
261
262
263
264
265
266
267
268
				gpuInfo := CudaGPUInfo{
					GpuInfo: GpuInfo{
						Library: "cuda",
					},
					index: i,
				}
				var driverMajor int
				var driverMinor int
Daniel Hiltgen's avatar
Daniel Hiltgen committed
269
270
				if cHandles.cudart != nil {
					C.cudart_bootstrap(*cHandles.cudart, C.int(i), &memInfo)
271
				} else {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
272
273
274
					C.nvcuda_bootstrap(*cHandles.nvcuda, C.int(i), &memInfo)
					driverMajor = int(cHandles.nvcuda.driver_major)
					driverMinor = int(cHandles.nvcuda.driver_minor)
275
276
277
278
279
280
281
282
283
284
				}
				if memInfo.err != nil {
					slog.Info("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
					C.free(unsafe.Pointer(memInfo.err))
					continue
				}
				gpuInfo.TotalMemory = uint64(memInfo.total)
				gpuInfo.FreeMemory = uint64(memInfo.free)
				gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
				gpuInfo.Compute = fmt.Sprintf("%d.%d", memInfo.major, memInfo.minor)
285
286
				gpuInfo.computeMajor = int(memInfo.major)
				gpuInfo.computeMinor = int(memInfo.minor)
287
				gpuInfo.MinimumMemory = cudaMinimumMemory
288
289
				gpuInfo.DriverMajor = driverMajor
				gpuInfo.DriverMinor = driverMinor
Daniel Hiltgen's avatar
Daniel Hiltgen committed
290
				variant := cudaVariant(gpuInfo)
291
292
293
				if depPath != "" {
					gpuInfo.DependencyPath = depPath
					// Check for variant specific directory
Daniel Hiltgen's avatar
Daniel Hiltgen committed
294
295
296
					if variant != "" {
						if _, err := os.Stat(filepath.Join(depPath, "cuda_"+variant)); err == nil {
							gpuInfo.DependencyPath = filepath.Join(depPath, "cuda_"+variant)
297
298
299
						}
					}
				}
300
				gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
Daniel Hiltgen's avatar
Daniel Hiltgen committed
301
				gpuInfo.Variant = variant
302

303
304
305
306
307
308
309
310
311
				if memInfo.major < CudaComputeMin[0] || (memInfo.major == CudaComputeMin[0] && memInfo.minor < CudaComputeMin[1]) {
					unsupportedGPUs = append(unsupportedGPUs,
						UnsupportedGPUInfo{
							GpuInfo: gpuInfo.GpuInfo,
						})
					slog.Info(fmt.Sprintf("[%d] CUDA GPU is too old. Compute Capability detected: %d.%d", i, memInfo.major, memInfo.minor))
					continue
				}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
				// query the management library as well so we can record any skew between the two
				// which represents overhead on the GPU we must set aside on subsequent updates
				if cHandles.nvml != nil {
					C.nvml_get_free(*cHandles.nvml, C.int(gpuInfo.index), &memInfo.free, &memInfo.total, &memInfo.used)
					if memInfo.err != nil {
						slog.Warn("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
						C.free(unsafe.Pointer(memInfo.err))
					} else {
						if memInfo.free != 0 && uint64(memInfo.free) > gpuInfo.FreeMemory {
							gpuInfo.OSOverhead = uint64(memInfo.free) - gpuInfo.FreeMemory
							slog.Info("detected OS VRAM overhead",
								"id", gpuInfo.ID,
								"library", gpuInfo.Library,
								"compute", gpuInfo.Compute,
								"driver", fmt.Sprintf("%d.%d", gpuInfo.DriverMajor, gpuInfo.DriverMinor),
								"name", gpuInfo.Name,
								"overhead", format.HumanBytes2(gpuInfo.OSOverhead),
							)
						}
					}
				}

334
335
				// TODO potentially sort on our own algorithm instead of what the underlying GPU library does...
				cudaGPUs = append(cudaGPUs, gpuInfo)
Wang,Zhe's avatar
Wang,Zhe committed
336
			}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
337
338
339
		}

		// Intel
Michael Yang's avatar
bool  
Michael Yang committed
340
		if envconfig.IntelGPU() {
341
			oHandles = initOneAPIHandles()
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
			if oHandles != nil && oHandles.oneapi != nil {
				for d := range oHandles.oneapi.num_drivers {
					if oHandles.oneapi == nil {
						// shouldn't happen
						slog.Warn("nil oneapi handle with driver count", "count", int(oHandles.oneapi.num_drivers))
						continue
					}
					devCount := C.oneapi_get_device_count(*oHandles.oneapi, C.int(d))
					for i := range devCount {
						gpuInfo := OneapiGPUInfo{
							GpuInfo: GpuInfo{
								Library: "oneapi",
							},
							driverIndex: int(d),
							gpuIndex:    int(i),
						}
						// TODO - split bootstrapping from updating free memory
						C.oneapi_check_vram(*oHandles.oneapi, C.int(d), i, &memInfo)
						// TODO - convert this to MinimumMemory based on testing...
						var totalFreeMem float64 = float64(memInfo.free) * 0.95 // work-around: leave some reserve vram for mkl lib used in ggml-sycl backend.
						memInfo.free = C.uint64_t(totalFreeMem)
						gpuInfo.TotalMemory = uint64(memInfo.total)
						gpuInfo.FreeMemory = uint64(memInfo.free)
						gpuInfo.ID = C.GoString(&memInfo.gpu_id[0])
						gpuInfo.Name = C.GoString(&memInfo.gpu_name[0])
						gpuInfo.DependencyPath = depPath
						oneapiGPUs = append(oneapiGPUs, gpuInfo)
369
					}
370
371
372
373
				}
			}
		}

374
375
376
377
		rocmGPUs, err = AMDGetGPUInfo()
		if err != nil {
			bootstrapErrors = append(bootstrapErrors, err)
		}
378
		bootstrapped = true
379
380
381
		if len(cudaGPUs) == 0 && len(rocmGPUs) == 0 && len(oneapiGPUs) == 0 {
			slog.Info("no compatible GPUs were discovered")
		}
382
383
384
385
386
387
	}

	// For detected GPUs, load library if not loaded

	// Refresh free memory usage
	if needRefresh {
388
389
390
391
392
393
394
395
396
		mem, err := GetCPUMem()
		if err != nil {
			slog.Warn("error looking up system memory", "error", err)
		} else {
			slog.Debug("updating system memory data",
				slog.Group(
					"before",
					"total", format.HumanBytes2(cpus[0].TotalMemory),
					"free", format.HumanBytes2(cpus[0].FreeMemory),
397
					"free_swap", format.HumanBytes2(cpus[0].FreeSwap),
398
399
400
401
402
				),
				slog.Group(
					"now",
					"total", format.HumanBytes2(mem.TotalMemory),
					"free", format.HumanBytes2(mem.FreeMemory),
403
					"free_swap", format.HumanBytes2(mem.FreeSwap),
404
405
406
				),
			)
			cpus[0].FreeMemory = mem.FreeMemory
407
			cpus[0].FreeSwap = mem.FreeSwap
408
409
		}

410
		var memInfo C.mem_info_t
Daniel Hiltgen's avatar
Daniel Hiltgen committed
411
412
		if cHandles == nil && len(cudaGPUs) > 0 {
			cHandles = initCudaHandles()
413
414
		}
		for i, gpu := range cudaGPUs {
415
416
417
			if cHandles.nvml != nil {
				C.nvml_get_free(*cHandles.nvml, C.int(gpu.index), &memInfo.free, &memInfo.total, &memInfo.used)
			} else if cHandles.cudart != nil {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
418
				C.cudart_bootstrap(*cHandles.cudart, C.int(gpu.index), &memInfo)
419
420
421
			} else if cHandles.nvcuda != nil {
				C.nvcuda_get_free(*cHandles.nvcuda, C.int(gpu.index), &memInfo.free, &memInfo.total)
				memInfo.used = memInfo.total - memInfo.free
Wang,Zhe's avatar
Wang,Zhe committed
422
			} else {
423
424
425
				// shouldn't happen
				slog.Warn("no valid cuda library loaded to refresh vram usage")
				break
Wang,Zhe's avatar
Wang,Zhe committed
426
427
			}
			if memInfo.err != nil {
428
				slog.Warn("error looking up nvidia GPU memory", "error", C.GoString(memInfo.err))
Wang,Zhe's avatar
Wang,Zhe committed
429
430
431
				C.free(unsafe.Pointer(memInfo.err))
				continue
			}
432
433
			if memInfo.free == 0 {
				slog.Warn("error looking up nvidia GPU memory")
Wang,Zhe's avatar
Wang,Zhe committed
434
435
				continue
			}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
436
437
438
439
			if cHandles.nvml != nil && gpu.OSOverhead > 0 {
				// When using the management library update based on recorded overhead
				memInfo.free -= C.uint64_t(gpu.OSOverhead)
			}
440
441
442
			slog.Debug("updating cuda memory data",
				"gpu", gpu.ID,
				"name", gpu.Name,
Daniel Hiltgen's avatar
Daniel Hiltgen committed
443
				"overhead", format.HumanBytes2(gpu.OSOverhead),
444
445
446
447
448
449
450
451
452
453
454
455
				slog.Group(
					"before",
					"total", format.HumanBytes2(gpu.TotalMemory),
					"free", format.HumanBytes2(gpu.FreeMemory),
				),
				slog.Group(
					"now",
					"total", format.HumanBytes2(uint64(memInfo.total)),
					"free", format.HumanBytes2(uint64(memInfo.free)),
					"used", format.HumanBytes2(uint64(memInfo.used)),
				),
			)
456
			cudaGPUs[i].FreeMemory = uint64(memInfo.free)
457
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474

		if oHandles == nil && len(oneapiGPUs) > 0 {
			oHandles = initOneAPIHandles()
		}
		for i, gpu := range oneapiGPUs {
			if oHandles.oneapi == nil {
				// shouldn't happen
				slog.Warn("nil oneapi handle with device count", "count", oHandles.deviceCount)
				continue
			}
			C.oneapi_check_vram(*oHandles.oneapi, C.int(gpu.driverIndex), C.int(gpu.gpuIndex), &memInfo)
			// TODO - convert this to MinimumMemory based on testing...
			var totalFreeMem float64 = float64(memInfo.free) * 0.95 // work-around: leave some reserve vram for mkl lib used in ggml-sycl backend.
			memInfo.free = C.uint64_t(totalFreeMem)
			oneapiGPUs[i].FreeMemory = uint64(memInfo.free)
		}

475
		err = RocmGPUInfoList(rocmGPUs).RefreshFreeMemory()
476
477
		if err != nil {
			slog.Debug("problem refreshing ROCm free memory", "error", err)
478
		}
479
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
480

481
482
483
484
485
486
487
	resp := []GpuInfo{}
	for _, gpu := range cudaGPUs {
		resp = append(resp, gpu.GpuInfo)
	}
	for _, gpu := range rocmGPUs {
		resp = append(resp, gpu.GpuInfo)
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
488
489
490
	for _, gpu := range oneapiGPUs {
		resp = append(resp, gpu.GpuInfo)
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
491
	if len(resp) == 0 {
492
		resp = append(resp, cpus[0].GpuInfo)
493
494
495
496
	}
	return resp
}

497
func FindGPULibs(baseLibName string, defaultPatterns []string) []string {
498
499
500
	// Multiple GPU libraries may exist, and some may not work, so keep trying until we exhaust them
	var ldPaths []string
	gpuLibPaths := []string{}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
501
	slog.Debug("Searching for GPU library", "name", baseLibName)
502

Daniel Hiltgen's avatar
Daniel Hiltgen committed
503
	// Start with our bundled libraries
Daniel Hiltgen's avatar
Daniel Hiltgen committed
504
	patterns := []string{filepath.Join(LibraryDir(), baseLibName)}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
505

506
507
508
509
510
511
512
513
	switch runtime.GOOS {
	case "windows":
		ldPaths = strings.Split(os.Getenv("PATH"), ";")
	case "linux":
		ldPaths = strings.Split(os.Getenv("LD_LIBRARY_PATH"), ":")
	default:
		return gpuLibPaths
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
514
515

	// Then with whatever we find in the PATH/LD_LIBRARY_PATH
516
517
518
519
520
	for _, ldPath := range ldPaths {
		d, err := filepath.Abs(ldPath)
		if err != nil {
			continue
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
521
		patterns = append(patterns, filepath.Join(d, baseLibName))
522
	}
523
	patterns = append(patterns, defaultPatterns...)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
524
	slog.Debug("gpu library search", "globs", patterns)
525
	for _, pattern := range patterns {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
526
527
528
529

		// Nvidia PhysX known to return bogus results
		if strings.Contains(pattern, "PhysX") {
			slog.Debug("skipping PhysX cuda library path", "path", pattern)
530
			continue
Daniel Hiltgen's avatar
Daniel Hiltgen committed
531
		}
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
		// Ignore glob discovery errors
		matches, _ := filepath.Glob(pattern)
		for _, match := range matches {
			// Resolve any links so we don't try the same lib multiple times
			// and weed out any dups across globs
			libPath := match
			tmp := match
			var err error
			for ; err == nil; tmp, err = os.Readlink(libPath) {
				if !filepath.IsAbs(tmp) {
					tmp = filepath.Join(filepath.Dir(libPath), tmp)
				}
				libPath = tmp
			}
			new := true
			for _, cmp := range gpuLibPaths {
				if cmp == libPath {
					new = false
					break
				}
			}
			if new {
				gpuLibPaths = append(gpuLibPaths, libPath)
			}
		}
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
558
	slog.Debug("discovered GPU libraries", "paths", gpuLibPaths)
559
560
561
	return gpuLibPaths
}

562
563
564
// Bootstrap the runtime library
// Returns: num devices, handle, libPath, error
func loadCUDARTMgmt(cudartLibPaths []string) (int, *C.cudart_handle_t, string, error) {
565
	var resp C.cudart_init_resp_t
566
	resp.ch.verbose = getVerboseState()
567
	var err error
568
	for _, libPath := range cudartLibPaths {
569
570
		lib := C.CString(libPath)
		defer C.free(unsafe.Pointer(lib))
571
		C.cudart_init(lib, &resp)
572
		if resp.err != nil {
573
574
			err = fmt.Errorf("Unable to load cudart library %s: %s", libPath, C.GoString(resp.err))
			slog.Debug(err.Error())
575
576
			C.free(unsafe.Pointer(resp.err))
		} else {
577
578
			err = nil
			return int(resp.num_devices), &resp.ch, libPath, err
579
580
		}
	}
581
	return 0, nil, "", err
582
583
}

584
585
586
// Bootstrap the driver library
// Returns: num devices, handle, libPath, error
func loadNVCUDAMgmt(nvcudaLibPaths []string) (int, *C.nvcuda_handle_t, string, error) {
587
588
	var resp C.nvcuda_init_resp_t
	resp.ch.verbose = getVerboseState()
589
	var err error
590
591
592
593
594
	for _, libPath := range nvcudaLibPaths {
		lib := C.CString(libPath)
		defer C.free(unsafe.Pointer(lib))
		C.nvcuda_init(lib, &resp)
		if resp.err != nil {
595
596
597
			// Decide what log level based on the type of error message to help users understand why
			switch resp.cudaErr {
			case C.CUDA_ERROR_INSUFFICIENT_DRIVER, C.CUDA_ERROR_SYSTEM_DRIVER_MISMATCH:
598
599
				err = fmt.Errorf("version mismatch between driver and cuda driver library - reboot or upgrade may be required: library %s", libPath)
				slog.Warn(err.Error())
600
			case C.CUDA_ERROR_NO_DEVICE:
601
602
				err = fmt.Errorf("no nvidia devices detected by library %s", libPath)
				slog.Info(err.Error())
603
			case C.CUDA_ERROR_UNKNOWN:
604
605
				err = fmt.Errorf("unknown error initializing cuda driver library %s: %s. see https://github.com/ollama/ollama/blob/main/docs/troubleshooting.md for more information", libPath, C.GoString(resp.err))
				slog.Warn(err.Error())
606
			default:
607
				msg := C.GoString(resp.err)
608
609
610
				if strings.Contains(msg, "wrong ELF class") {
					slog.Debug("skipping 32bit library", "library", libPath)
				} else {
611
612
					err = fmt.Errorf("Unable to load cudart library %s: %s", libPath, C.GoString(resp.err))
					slog.Info(err.Error())
613
614
				}
			}
615
616
			C.free(unsafe.Pointer(resp.err))
		} else {
617
618
			err = nil
			return int(resp.num_devices), &resp.ch, libPath, err
619
620
		}
	}
621
	return 0, nil, "", err
622
623
}

624
625
626
// Bootstrap the management library
// Returns: handle, libPath, error
func loadNVMLMgmt(nvmlLibPaths []string) (*C.nvml_handle_t, string, error) {
627
628
	var resp C.nvml_init_resp_t
	resp.ch.verbose = getVerboseState()
629
	var err error
630
631
632
633
634
	for _, libPath := range nvmlLibPaths {
		lib := C.CString(libPath)
		defer C.free(unsafe.Pointer(lib))
		C.nvml_init(lib, &resp)
		if resp.err != nil {
635
636
			err = fmt.Errorf("Unable to load NVML management library %s: %s", libPath, C.GoString(resp.err))
			slog.Info(err.Error())
637
638
			C.free(unsafe.Pointer(resp.err))
		} else {
639
640
			err = nil
			return &resp.ch, libPath, err
641
642
		}
	}
643
	return nil, "", err
644
645
}

646
647
648
// bootstrap the Intel GPU library
// Returns: num devices, handle, libPath, error
func loadOneapiMgmt(oneapiLibPaths []string) (int, *C.oneapi_handle_t, string, error) {
Wang,Zhe's avatar
Wang,Zhe committed
649
	var resp C.oneapi_init_resp_t
Daniel Hiltgen's avatar
Daniel Hiltgen committed
650
	num_devices := 0
Wang,Zhe's avatar
Wang,Zhe committed
651
	resp.oh.verbose = getVerboseState()
652
	var err error
Wang,Zhe's avatar
Wang,Zhe committed
653
654
655
656
657
	for _, libPath := range oneapiLibPaths {
		lib := C.CString(libPath)
		defer C.free(unsafe.Pointer(lib))
		C.oneapi_init(lib, &resp)
		if resp.err != nil {
658
659
			err = fmt.Errorf("Unable to load oneAPI management library %s: %s", libPath, C.GoString(resp.err))
			slog.Debug(err.Error())
Wang,Zhe's avatar
Wang,Zhe committed
660
661
			C.free(unsafe.Pointer(resp.err))
		} else {
662
			err = nil
Daniel Hiltgen's avatar
Daniel Hiltgen committed
663
			for i := range resp.oh.num_drivers {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
664
665
				num_devices += int(C.oneapi_get_device_count(resp.oh, C.int(i)))
			}
666
			return num_devices, &resp.oh, libPath, err
Wang,Zhe's avatar
Wang,Zhe committed
667
668
		}
	}
669
	return 0, nil, "", err
Wang,Zhe's avatar
Wang,Zhe committed
670
671
}

672
func getVerboseState() C.uint16_t {
Michael Yang's avatar
Michael Yang committed
673
	if envconfig.Debug() {
674
675
676
677
		return C.uint16_t(1)
	}
	return C.uint16_t(0)
}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
678
679
680
681
682
683
684
685
686
687
688
689
690
691

// Given the list of GPUs this instantiation is targeted for,
// figure out the visible devices environment variable
//
// If different libraries are detected, the first one is what we use
func (l GpuInfoList) GetVisibleDevicesEnv() (string, string) {
	if len(l) == 0 {
		return "", ""
	}
	switch l[0].Library {
	case "cuda":
		return cudaGetVisibleDevicesEnv(l)
	case "rocm":
		return rocmGetVisibleDevicesEnv(l)
Wang,Zhe's avatar
Wang,Zhe committed
692
693
	case "oneapi":
		return oneapiGetVisibleDevicesEnv(l)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
694
695
696
697
698
	default:
		slog.Debug("no filter required for library " + l[0].Library)
		return "", ""
	}
}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
699

Daniel Hiltgen's avatar
Daniel Hiltgen committed
700
func LibraryDir() string {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
701
702
703
704
705
706
707
708
709
710
	// On Windows/linux we bundle the dependencies at the same level as the executable
	appExe, err := os.Executable()
	if err != nil {
		slog.Warn("failed to lookup executable path", "error", err)
	}
	cwd, err := os.Getwd()
	if err != nil {
		slog.Warn("failed to lookup working directory", "error", err)
	}
	// Scan for any of our dependeices, and pick first match
711
	for _, root := range []string{filepath.Dir(appExe), filepath.Join(filepath.Dir(appExe), envconfig.LibRelativeToExe()), cwd} {
712
		libDep := filepath.Join("lib", "ollama")
Daniel Hiltgen's avatar
Daniel Hiltgen committed
713
714
715
716
717
718
719
720
721
722
723
724
725
726
		if _, err := os.Stat(filepath.Join(root, libDep)); err == nil {
			return filepath.Join(root, libDep)
		}
		// Developer mode, local build
		if _, err := os.Stat(filepath.Join(root, runtime.GOOS+"-"+runtime.GOARCH, libDep)); err == nil {
			return filepath.Join(root, runtime.GOOS+"-"+runtime.GOARCH, libDep)
		}
		if _, err := os.Stat(filepath.Join(root, "dist", runtime.GOOS+"-"+runtime.GOARCH, libDep)); err == nil {
			return filepath.Join(root, "dist", runtime.GOOS+"-"+runtime.GOARCH, libDep)
		}
	}
	slog.Warn("unable to locate gpu dependency libraries")
	return ""
}
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746

func GetSystemInfo() SystemInfo {
	gpus := GetGPUInfo()
	gpuMutex.Lock()
	defer gpuMutex.Unlock()
	discoveryErrors := []string{}
	for _, err := range bootstrapErrors {
		discoveryErrors = append(discoveryErrors, err.Error())
	}
	if len(gpus) == 1 && gpus[0].Library == "cpu" {
		gpus = []GpuInfo{}
	}

	return SystemInfo{
		System:          cpus[0],
		GPUs:            gpus,
		UnsupportedGPUs: unsupportedGPUs,
		DiscoveryErrors: discoveryErrors,
	}
}