memory.go 15.5 KB
Newer Older
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1
2
3
package llm

import (
4
	"fmt"
Daniel Hiltgen's avatar
Daniel Hiltgen committed
5
	"log/slog"
6
	"os"
7
	"slices"
Jesse Gross's avatar
Jesse Gross committed
8
	"sort"
9
	"strings"
Daniel Hiltgen's avatar
Daniel Hiltgen committed
10
11

	"github.com/ollama/ollama/api"
12
	"github.com/ollama/ollama/envconfig"
Daniel Hiltgen's avatar
Daniel Hiltgen committed
13
	"github.com/ollama/ollama/format"
Michael Yang's avatar
Michael Yang committed
14
	"github.com/ollama/ollama/fs/ggml"
15
	"github.com/ollama/ollama/ml"
Daniel Hiltgen's avatar
Daniel Hiltgen committed
16
17
)

Jesse Gross's avatar
Jesse Gross committed
18
19
20
// pickBestFullFitByLibrary will try to find the optimal placement of the model in the available GPUs where the model fully fits
// The list of GPUs returned will always be the same brand (library)
// If the model can not be fit fully within the available GPU(s) nil is returned
21
22
23
func pickBestFullFitByLibrary(f *ggml.GGML, modelPath string, projectors []string, adapters []string, opts api.Options, gpus []ml.DeviceInfo, numParallel int) []ml.DeviceInfo {
	for _, gl := range ml.ByLibrary(gpus) {
		sgl := append(make([]ml.DeviceInfo, 0, len(gl)), gl...)
Jesse Gross's avatar
Jesse Gross committed
24
25
26
27

		// TODO - potentially sort by performance capability, existing models loaded, etc.
		// TODO - Eliminate any GPUs that already have envconfig.MaxRunners loaded on them
		// Note: at present, this will favor most current available VRAM descending and ignoring faster GPU speed in mixed setups
28
		sort.Sort(sort.Reverse(ml.ByFreeMemory(sgl)))
Jesse Gross's avatar
Jesse Gross committed
29
30
31
32
33

		if !envconfig.SchedSpread() {
			// Try to pack into as few GPUs as possible, starting from 1 GPU
			for numGPUs := 1; numGPUs <= len(sgl); numGPUs++ {
				gpuSubset := sgl[:numGPUs]
34
				ok, estimatedVRAM := predictServerFit(gpuSubset, f, adapters, projectors, opts, numParallel)
Jesse Gross's avatar
Jesse Gross committed
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

				if ok {
					slog.Info("new model will fit in available VRAM across minimum required GPUs, loading",
						"model", modelPath,
						"library", sgl[0].Library,
						"parallel", numParallel,
						"required", format.HumanBytes2(estimatedVRAM),
						"gpus", numGPUs)
					return gpuSubset
				}
			}
		} else {
			// TODO future refinements
			// - if multiple Libraries, see if any single GPU in any Library will fit
			// - try subsets of GPUs instead of just falling back to 1 or all in a family

			// Now try all the GPUS (OLLAMA_SCHED_SPREAD is set)
52
			if ok, estimatedVRAM := predictServerFit(sgl, f, adapters, projectors, opts, numParallel); ok {
Jesse Gross's avatar
Jesse Gross committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
				slog.Info("new model will fit in available VRAM, loading",
					"model", modelPath,
					"library", sgl[0].Library,
					"parallel", numParallel,
					"required", format.HumanBytes2(estimatedVRAM),
					"gpus", len(sgl))
				return sgl
			}
		}
	}
	return nil
}

// If multiple Libraries are detected, pick the Library which loads the most layers for the model
67
68
func pickBestPartialFitByLibrary(f *ggml.GGML, projectors []string, adapters []string, opts api.Options, gpus []ml.DeviceInfo, numParallel int) []ml.DeviceInfo {
	byLibrary := ml.ByLibrary(gpus)
Jesse Gross's avatar
Jesse Gross committed
69
70
71
72
73
74
	if len(byLibrary) <= 1 {
		return gpus
	}
	var bestEstimate uint64
	var bestFit int
	for i, gl := range byLibrary {
75
		_, estimatedVRAM := predictServerFit(gl, f, adapters, projectors, opts, numParallel)
Jesse Gross's avatar
Jesse Gross committed
76
77
78
79
80
81
82
83
		if estimatedVRAM > bestEstimate {
			bestEstimate = estimatedVRAM
			bestFit = i
		}
	}
	return byLibrary[bestFit]
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
84
// This algorithm looks for a complete fit to determine if we need to unload other models
85
func predictServerFit(allGpus []ml.DeviceInfo, f *ggml.GGML, adapters, projectors []string, opts api.Options, numParallel int) (bool, uint64) {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
86
	// Split up the GPUs by type and try them
87
	var estimatedVRAM uint64
88
	for _, gpus := range ml.ByLibrary(allGpus) {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
89
		var layerCount int
Jesse Gross's avatar
Jesse Gross committed
90
		estimate := estimateGPULayers(gpus, f, projectors, opts, numParallel)
91
		layerCount, estimatedVRAM = estimate.Layers, estimate.VRAMSize
Daniel Hiltgen's avatar
Daniel Hiltgen committed
92
		if opts.NumGPU < 0 {
Michael Yang's avatar
Michael Yang committed
93
			if layerCount > 0 && layerCount >= int(f.KV().BlockCount()+1) {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
94
95
96
97
98
99
100
101
102
103
104
				return true, estimatedVRAM
			}
		} else {
			if layerCount > 0 && layerCount >= opts.NumGPU {
				return true, estimatedVRAM
			}
		}
	}
	return false, estimatedVRAM
}

105
106
107
108
109
110
111
112
113
114
115
116
117
func verifyCPUFit(f *ggml.GGML, modelPath string, projectors []string, adapters []string, opts api.Options, systemInfo ml.SystemInfo, numParallel int) bool {
	estimate := estimateGPULayers(nil, f, projectors, opts, numParallel)
	if estimate.TotalSize > systemInfo.FreeMemory {
		return false
	}
	slog.Info("new model will fit in available system memory for CPU inference, loading",
		"model", modelPath,
		"parallel", numParallel,
		"required", format.HumanBytes2(estimate.TotalSize),
	)
	return true
}

118
119
120
121
122
123
124
125
126
127
128
129
130
131
type MemoryEstimate struct {
	// How many layers we predict we can load
	Layers int

	// The size of the graph which occupies the main GPU
	Graph uint64

	// How much VRAM will be allocated given the number of layers we predict
	VRAMSize uint64

	// The total size of the model if loaded into VRAM.  If all layers are loaded, VRAMSize == TotalSize
	TotalSize uint64

	// For multi-GPU scenarios, this provides the tensor split parameter
Jesse Gross's avatar
Jesse Gross committed
132
	TensorSplit []int
133
134
135

	// For multi-GPU scenarios, this is the size in bytes per GPU
	GPUSizes []uint64
136
137
138
139
140
141
142
143
144
145
146
147

	// internal fields for logging purposes
	inferenceLibrary    string
	layersRequested     int
	layersModel         int
	availableList       []string
	kv                  uint64
	allocationsList     []string
	memoryWeights       uint64
	memoryLayerOutput   uint64
	graphFullOffload    uint64
	graphPartialOffload uint64
148
149

	projectorWeights, projectorGraph uint64
150
151
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
152
// Given a model and one or more GPU targets, predict how many layers and bytes we can load, and the total size
Daniel Hiltgen's avatar
Daniel Hiltgen committed
153
// The GPUs provided must all be the same Library
154
func estimateGPULayers(gpus []ml.DeviceInfo, f *ggml.GGML, projectors []string, opts api.Options, numParallel int) MemoryEstimate {
155
156
157
158
159
160
161
162
163
164
	// Graph size for a partial offload, applies to all GPUs
	var graphPartialOffload uint64

	// Graph size when all layers are offloaded, applies to all GPUs
	var graphFullOffload uint64

	// Final graph offload once we know full or partial
	var graphOffload uint64

	// Projectors loaded into GPU0 only
165
166
167
168
169
	var llamaEngineProjectorWeights uint64

	// Projectors loaded with output layer
	var ollamaEngineProjectorWeights uint64
	var ollamaEngineProjectorGraph uint64
170
171
172
173

	// Conditional output size on GPU 0
	var memoryLayerOutput uint64

Daniel Hiltgen's avatar
Daniel Hiltgen committed
174
175
	// The sizes of a layer
	var layerSize uint64
Daniel Hiltgen's avatar
Daniel Hiltgen committed
176

177
178
179
180
181
182
183
184
185
	// The sum of all the layer sizes (just for logging)
	var memoryWeights uint64

	// True if all the layers are loaded
	var fullyLoaded bool

	// Overflow that didn't fit into the GPU
	var overflow uint64

186
	overhead := envconfig.GpuOverhead()
187
	availableList := make([]string, len(gpus))
188
	libraries := []string{}
189
190
	for i, gpu := range gpus {
		availableList[i] = format.HumanBytes2(gpu.FreeMemory)
191
192
193
194
195
196
		if !slices.Contains(libraries, gpu.Library) {
			libraries = append(libraries, gpu.Library)
		}
	}
	if len(libraries) == 0 {
		libraries = []string{"cpu"}
197
	}
198
	slog.Debug("evaluating", "library", strings.Join(libraries, ","), "gpu_count", len(gpus), "available", availableList)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
199
200

	for _, projector := range projectors {
201
		llamaEngineProjectorWeights += projectorMemoryRequirements(projector)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
202
	}
203
204
	if llamaEngineProjectorWeights == 0 {
		ollamaEngineProjectorWeights, ollamaEngineProjectorGraph = f.VisionGraphSize()
205
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
206

Michael Yang's avatar
Michael Yang committed
207
	layers := f.Tensors().GroupLayers()
Michael Yang's avatar
typo  
Michael Yang committed
208
209
	// add one layer worth of memory as a buffer
	if blk0, ok := layers["blk.0"]; ok {
Michael Yang's avatar
Michael Yang committed
210
		layerSize = blk0.Size()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
211
212
	} else {
		slog.Warn("model missing blk.0 layer size")
Michael Yang's avatar
typo  
Michael Yang committed
213
	}
Michael Yang's avatar
Michael Yang committed
214

215
	useFlashAttention := envconfig.FlashAttention(f.FlashAttention()) &&
216
		ml.FlashAttentionSupported(gpus) &&
217
218
219
220
		f.SupportsFlashAttention()

	var kvct string
	if useFlashAttention {
221
		requested := strings.ToLower(envconfig.KvCacheType())
222
		if f.SupportsKVCacheType(requested) {
223
224
225
226
			kvct = requested
		}
	}

227
	kv, graphPartialOffload, graphFullOffload := f.GraphSize(uint64(opts.NumCtx), uint64(min(opts.NumCtx, opts.NumBatch)), numParallel, kvct, useFlashAttention)
228

229
230
231
232
233
234
235
236
	if len(kv) > 0 {
		layerSize += kv[0]
	}

	var kvTotal uint64
	for _, kvLayer := range kv {
		kvTotal += kvLayer
	}
237

Daniel Hiltgen's avatar
Daniel Hiltgen committed
238
	if graphPartialOffload == 0 {
239
240
241
242
243
244
		headsKV := f.KV().HeadCountKVMin()
		if headsKV == 0 {
			headsKV = 1
		}
		gqa := f.KV().HeadCountMax() / headsKV
		graphPartialOffload = gqa * kvTotal / 6
Daniel Hiltgen's avatar
Daniel Hiltgen committed
245
246
247
248
249
	}
	if graphFullOffload == 0 {
		graphFullOffload = graphPartialOffload
	}

250
	// on metal there's no partial offload overhead
251
	if len(gpus) > 0 && gpus[0].Library == "Metal" {
252
		graphPartialOffload = graphFullOffload
Daniel Hiltgen's avatar
Daniel Hiltgen committed
253
254
255
	} else if len(gpus) > 1 {
		// multigpu should always use the partial graph size
		graphFullOffload = graphPartialOffload
256
257
	}

258
	// Output layer handled at the end if we have space
259
	if layer, ok := layers["output_norm"]; ok {
Michael Yang's avatar
Michael Yang committed
260
		memoryLayerOutput += layer.Size()
261
262
	}
	if layer, ok := layers["output"]; ok {
Michael Yang's avatar
Michael Yang committed
263
		memoryLayerOutput += layer.Size()
264
	} else if layer, ok := layers["token_embd"]; ok {
Michael Yang's avatar
Michael Yang committed
265
		memoryLayerOutput += layer.Size()
Michael Yang's avatar
Michael Yang committed
266
267
	}

268
	gpuZeroOverhead := llamaEngineProjectorWeights
269
270

	// Reduce set of GPUs to only those that have sufficient space to fit overhead and at least one layer
Michael Yang's avatar
Michael Yang committed
271
	var layerCount int
Jesse Gross's avatar
Jesse Gross committed
272
	tensorSplit := make([]int, len(gpus))
273
274
275
	gpuAllocations := make([]uint64, len(gpus))
	type gs struct {
		i int
276
		g *ml.DeviceInfo
277
278
279
280
281
282
283
284
	}
	gpusWithSpace := []gs{}
	for i := range gpus {
		var gzo uint64
		if len(gpusWithSpace) == 0 {
			gzo = gpuZeroOverhead
		}
		// Only include GPUs that can fit the graph, gpu minimum, the layer buffer and at least more layer
285
		if gpus[i].FreeMemory < overhead+gzo+max(graphPartialOffload, graphFullOffload)+gpus[i].MinimumMemory()+2*layerSize {
286
287
288
			slog.Debug("gpu has too little memory to allocate any layers",
				"id", gpus[i].ID,
				"library", gpus[i].Library,
289
				"compute", gpus[i].Compute(),
290
291
292
293
294
295
296
297
298
299
				"driver", fmt.Sprintf("%d.%d", gpus[i].DriverMajor, gpus[i].DriverMinor),
				"name", gpus[i].Name,
				"total", format.HumanBytes2(gpus[i].TotalMemory),
				"available", format.HumanBytes2(gpus[i].FreeMemory),
				"minimum_memory", gpus[i].MinimumMemory,
				"layer_size", format.HumanBytes2(layerSize),
				"gpu_zer_overhead", format.HumanBytes2(gzo),
				"partial_offload", format.HumanBytes2(graphPartialOffload),
				"full_offload", format.HumanBytes2(graphFullOffload),
			)
300
301
302
			continue
		}
		gpusWithSpace = append(gpusWithSpace, gs{i, &gpus[i]})
303
		gpuAllocations[i] += gpus[i].MinimumMemory() + layerSize // We hold off on graph until we know partial vs. full
304
305
306
307
308
309
	}

	var gpuZeroID int
	if len(gpusWithSpace) > 0 {
		gpuZeroID = gpusWithSpace[0].i
		gpuAllocations[gpuZeroID] += gpuZeroOverhead
310
311
	} else {
		overflow += gpuZeroOverhead
312
313
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
314
	// For all the layers, find where they can fit on the GPU(s)
315
	for i := int(f.KV().BlockCount()) - 1; i >= 0; i-- {
316
317
		// Some models have inconsistent layer sizes
		if blk, ok := layers[fmt.Sprintf("blk.%d", i)]; ok {
Michael Yang's avatar
Michael Yang committed
318
			layerSize = blk.Size()
319
			layerSize += kv[i]
Michael Yang's avatar
Michael Yang committed
320
			memoryWeights += blk.Size()
321
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
322

323
324
		if opts.NumGPU >= 0 && layerCount >= opts.NumGPU {
			// Stop allocating on GPU(s) once we hit the users target NumGPU
325
			overflow += layerSize
326
327
328
329
330
331
332
			continue
		}

		// distribute the layers across the GPU(s) that have space
		for j := len(gpusWithSpace); j > 0; j-- {
			g := gpusWithSpace[i%j]
			used := gpuAllocations[g.i] + max(graphPartialOffload, graphFullOffload)
333
			if g.g.FreeMemory > overhead+used+layerSize {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
334
				gpuAllocations[g.i] += layerSize
Jesse Gross's avatar
Jesse Gross committed
335
				tensorSplit[g.i]++
Michael Yang's avatar
typo  
Michael Yang committed
336
				layerCount++
337
338
339
				break
			} else {
				gpusWithSpace = append(gpusWithSpace[:i%j], gpusWithSpace[i%j+1:]...)
Michael Yang's avatar
typo  
Michael Yang committed
340
			}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
341
		}
342
343
344
345

		if len(gpusWithSpace) == 0 {
			overflow += layerSize
		}
346
	}
Michael Yang's avatar
Michael Yang committed
347
	if layerCount >= int(f.KV().BlockCount()) {
348
349
		fullyLoaded = true
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
350
351

	// Determine if we need to consider output then find where it fits
352
353
	memoryLastLayer := memoryLayerOutput + ollamaEngineProjectorWeights + ollamaEngineProjectorGraph
	if memoryLastLayer > 0 {
354
355
356
357
		if opts.NumGPU < 0 || layerCount < opts.NumGPU {
			for j := len(gpusWithSpace); j > 0; j-- {
				g := gpusWithSpace[layerCount%j]
				used := gpuAllocations[g.i] + max(graphPartialOffload, graphFullOffload)
358
359
				if g.g.FreeMemory > overhead+used+memoryLastLayer {
					gpuAllocations[g.i] += memoryLastLayer
Jesse Gross's avatar
Jesse Gross committed
360
					tensorSplit[g.i]++
361
362
363
					layerCount++
					break
				}
364
365
			}
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
366

Michael Yang's avatar
Michael Yang committed
367
		if layerCount < int(f.KV().BlockCount())+1 {
368
			fullyLoaded = false
369
			overflow += memoryLastLayer
370
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
371
372
	}

373
374
	// Add the applicable (full or partial) graph allocations
	for i := range gpus {
Jesse Gross's avatar
Jesse Gross committed
375
		if tensorSplit[i] <= 0 {
376
377
378
379
380
381
382
383
384
385
386
387
			continue
		}
		if fullyLoaded {
			gpuAllocations[i] += graphFullOffload
		} else {
			gpuAllocations[i] += graphPartialOffload
		}
	}
	if fullyLoaded {
		graphOffload = graphFullOffload
	} else {
		graphOffload = graphPartialOffload
Daniel Hiltgen's avatar
Daniel Hiltgen committed
388
389
	}

390
391
392
393
	// Summaries for the log
	var memoryRequiredPartial, memoryRequiredTotal uint64
	for i := range gpuAllocations {
		memoryRequiredPartial += gpuAllocations[i]
Daniel Hiltgen's avatar
Daniel Hiltgen committed
394
	}
395
	memoryRequiredTotal = memoryRequiredPartial + overflow
Daniel Hiltgen's avatar
Daniel Hiltgen committed
396

397
398
399
400
	allocationsList := []string{}
	for _, a := range gpuAllocations {
		allocationsList = append(allocationsList, format.HumanBytes2(a))
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
401

402
403
404
405
406
407
408
	estimate := MemoryEstimate{
		TotalSize: memoryRequiredTotal,
		Layers:    0,
		Graph:     0,
		VRAMSize:  0,
		GPUSizes:  []uint64{},

409
		inferenceLibrary:    strings.Join(libraries, ","),
410
		layersRequested:     opts.NumGPU,
Michael Yang's avatar
Michael Yang committed
411
		layersModel:         int(f.KV().BlockCount()) + 1,
412
		availableList:       availableList,
413
		kv:                  kvTotal,
414
415
416
417
418
		allocationsList:     allocationsList,
		memoryWeights:       memoryWeights,
		memoryLayerOutput:   memoryLayerOutput,
		graphFullOffload:    graphFullOffload,
		graphPartialOffload: graphPartialOffload,
419
420
		projectorWeights:    llamaEngineProjectorWeights + ollamaEngineProjectorWeights,
		projectorGraph:      ollamaEngineProjectorGraph,
421
422
	}

423
	if len(gpus) == 0 {
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
		return estimate
	}
	if layerCount == 0 {
		slog.Debug("insufficient VRAM to load any model layers")
		return estimate
	}
	estimate.Layers = layerCount
	estimate.Graph = graphOffload
	estimate.VRAMSize = memoryRequiredPartial
	estimate.TotalSize = memoryRequiredTotal
	estimate.TensorSplit = tensorSplit
	estimate.GPUSizes = gpuAllocations
	return estimate
}

Michael Yang's avatar
Michael Yang committed
439
440
441
func (m MemoryEstimate) LogValue() slog.Value {
	attrs := []slog.Attr{
		slog.String("library", m.inferenceLibrary),
Daniel Hiltgen's avatar
Daniel Hiltgen committed
442
443
		slog.Group(
			"layers",
Michael Yang's avatar
Michael Yang committed
444
			// requested number of layers to offload
445
			"requested", m.layersRequested,
446
			// The number of layers the model has (including output)
447
			"model", m.layersModel,
Daniel Hiltgen's avatar
Daniel Hiltgen committed
448
			// estimated number of layers that can be offloaded
449
450
451
			"offload", m.Layers,
			// multi-gpu split for tensors
			"split", m.TensorSplit,
Daniel Hiltgen's avatar
Daniel Hiltgen committed
452
453
454
		),
		slog.Group(
			"memory",
455
			// memory available by GPU for offloading
456
			"available", m.availableList,
Michael Yang's avatar
Michael Yang committed
457
			"gpu_overhead", format.HumanBytes2(envconfig.GpuOverhead()),
Daniel Hiltgen's avatar
Daniel Hiltgen committed
458
459
460
			slog.Group(
				"required",
				// memory required for full offloading
461
				"full", format.HumanBytes2(m.TotalSize),
Daniel Hiltgen's avatar
Daniel Hiltgen committed
462
				// memory required to offload layers.estimate layers
463
				"partial", format.HumanBytes2(m.VRAMSize),
Daniel Hiltgen's avatar
Daniel Hiltgen committed
464
				// memory of KV cache
465
				"kv", format.HumanBytes2(m.kv),
466
				// Allocations across the GPUs
467
				"allocations", m.allocationsList,
Daniel Hiltgen's avatar
Daniel Hiltgen committed
468
469
470
471
			),
			slog.Group(
				"weights",
				// memory of the weights
472
				"total", format.HumanBytes2(m.memoryWeights+m.memoryLayerOutput),
Daniel Hiltgen's avatar
Daniel Hiltgen committed
473
				// memory of repeating layers
Michael Yang's avatar
Michael Yang committed
474
				"repeating", format.HumanBytes2(m.memoryWeights),
Daniel Hiltgen's avatar
Daniel Hiltgen committed
475
				// memory of non-repeating layers
476
				"nonrepeating", format.HumanBytes2(m.memoryLayerOutput),
Daniel Hiltgen's avatar
Daniel Hiltgen committed
477
478
479
480
			),
			slog.Group(
				"graph",
				// memory of graph when fully offloaded
481
				"full", format.HumanBytes2(m.graphFullOffload),
Daniel Hiltgen's avatar
Daniel Hiltgen committed
482
				// memory of graph when not fully offloaded
483
				"partial", format.HumanBytes2(m.graphPartialOffload),
Daniel Hiltgen's avatar
Daniel Hiltgen committed
484
485
			),
		),
Michael Yang's avatar
Michael Yang committed
486
487
488
489
490
491
492
493
494
495
496
	}

	if m.projectorWeights > 0 {
		attrs = append(attrs, slog.Group(
			"projector",
			"weights", format.HumanBytes2(m.projectorWeights),
			"graph", format.HumanBytes2(m.projectorGraph),
		))
	}

	return slog.GroupValue(attrs...)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
497
}
498

499
func projectorMemoryRequirements(filename string) (weights uint64) {
500
501
	file, err := os.Open(filename)
	if err != nil {
502
		return 0
503
504
505
	}
	defer file.Close()

506
	ggml, err := ggml.Decode(file, 1024)
507
	if err != nil {
508
		return 0
509
510
	}

Michael Yang's avatar
Michael Yang committed
511
512
	for _, layer := range ggml.Tensors().GroupLayers() {
		weights += layer.Size()
513
514
	}

515
	return weights
516
}