ggml.go 25.3 KB
Newer Older
Michael Yang's avatar
Michael Yang committed
1
2
package ggml

3
4
5
6
7
8
// #cgo CPPFLAGS: -I${SRCDIR}/ggml/include
// #include <stdlib.h>
// #include <stdint.h>
// #include "ggml.h"
// #include "ggml-cpu.h"
// #include "ggml-backend.h"
Michael Yang's avatar
Michael Yang committed
9
10
11
import "C"

import (
12
	"context"
Michael Yang's avatar
Michael Yang committed
13
14
15
	"fmt"
	"io"
	"log/slog"
16
	"maps"
Michael Yang's avatar
Michael Yang committed
17
	"os"
18
	"runtime"
19
20
21
	"slices"
	"strconv"
	"strings"
22
	"sync/atomic"
23
	"unicode"
Michael Yang's avatar
Michael Yang committed
24
25
26
27
28
	"unsafe"

	"github.com/ollama/ollama/format"
	fs "github.com/ollama/ollama/fs/ggml"
	"github.com/ollama/ollama/ml"
29
	ggml "github.com/ollama/ollama/ml/backend/ggml/ggml/src"
Michael Yang's avatar
Michael Yang committed
30
31
32
	"golang.org/x/sync/errgroup"
)

Michael Yang's avatar
Michael Yang committed
33
34
35
36
37
func devices() []*C.struct_ggml_backend_device {
	ggml.OnceLoad()
	ds := make([]*C.struct_ggml_backend_device, C.ggml_backend_dev_count())
	for i := range ds {
		ds[i] = C.ggml_backend_dev_get(C.size_t(i))
Michael Yang's avatar
Michael Yang committed
38
	}
Michael Yang's avatar
Michael Yang committed
39
40

	return ds
41
}
Michael Yang's avatar
Michael Yang committed
42
43

type Backend struct {
44
45
46
	meta    *fs.GGML
	sched   *C.struct_ggml_backend_sched
	tensors map[string]*C.struct_ggml_tensor
Michael Yang's avatar
Michael Yang committed
47
48

	// input is the backend used for inputs
49
	input *C.struct_ggml_backend_buffer_type
Michael Yang's avatar
Michael Yang committed
50
51

	// output is the backend used for outputs
52
	output *C.struct_ggml_backend_buffer_type
Michael Yang's avatar
Michael Yang committed
53
54

	// layers is the backend used for repeating layers
55
	layers map[int]*C.struct_ggml_backend_buffer_type
56

57
	flashAttention bool
Michael Yang's avatar
Michael Yang committed
58
59
60

	// maxGraphNodes is the maximum allowed number of graph nodes in this scheduler
	maxGraphNodes int
Michael Yang's avatar
Michael Yang committed
61
62
}

63
func New(ctx context.Context, r *os.File, params ml.BackendParams) (ml.Backend, error) {
Michael Yang's avatar
Michael Yang committed
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
	meta, n, err := fs.Decode(r, -1)
	if err != nil {
		return nil, err
	}

	slog.Info(
		"",
		"architecture", meta.KV().Architecture(),
		"file_type", meta.KV().FileType(),
		"name", meta.KV().String("general.name"),
		"description", meta.KV().String("general.description"),
		"num_tensors", len(meta.Tensors().Items()),
		"num_key_values", len(meta.KV()),
	)

79
	type deviceBufferType struct {
80
81
82
83
84
		d   *C.struct_ggml_backend_device
		bts []*C.struct_ggml_backend_buffer_type
	}

	var cpus, accels, gpus []*C.struct_ggml_backend_device
Michael Yang's avatar
Michael Yang committed
85
	for _, d := range devices() {
86
87
		switch C.ggml_backend_dev_type(d) {
		case C.GGML_BACKEND_DEVICE_TYPE_CPU:
88
89
90
91
			if len(cpus) == 0 {
				// only the first cpu device should be used
				cpus = append(cpus, d)
			}
92
93
		case C.GGML_BACKEND_DEVICE_TYPE_ACCEL:
			accels = append(accels, d)
Michael Yang's avatar
Michael Yang committed
94
		case C.GGML_BACKEND_DEVICE_TYPE_GPU:
95
			gpus = append(gpus, d)
Michael Yang's avatar
Michael Yang committed
96
97
98
		}
	}

Michael Yang's avatar
Michael Yang committed
99
	// create list of buffer types for the cpu
Michael Yang's avatar
Michael Yang committed
100
	cpuDeviceBufferType := deviceBufferType{d: C.ggml_backend_dev_by_type(C.GGML_BACKEND_DEVICE_TYPE_CPU)}
101
102
103
104
	for _, d := range append(accels, append(gpus, cpus...)...) {
		switch C.ggml_backend_dev_type(d) {
		case C.GGML_BACKEND_DEVICE_TYPE_CPU,
			C.GGML_BACKEND_DEVICE_TYPE_ACCEL:
Michael Yang's avatar
Michael Yang committed
105
			cpuDeviceBufferType.bts = append(cpuDeviceBufferType.bts, C.ggml_backend_dev_buffer_type(d))
Michael Yang's avatar
Michael Yang committed
106
		}
107
108
	}

Michael Yang's avatar
Michael Yang committed
109
	// create list of buffer types for each gpu
110
	var gpuDeviceBufferTypes []deviceBufferType
111
112
	for _, d := range gpus {
		bt := C.ggml_backend_dev_buffer_type(d)
113
		gpuDeviceBufferTypes = append(gpuDeviceBufferTypes, deviceBufferType{
114
			d:   d,
Michael Yang's avatar
Michael Yang committed
115
			bts: append([]*C.struct_ggml_backend_buffer_type{bt}, cpuDeviceBufferType.bts...),
116
		})
Michael Yang's avatar
Michael Yang committed
117
118
	}

Michael Yang's avatar
Michael Yang committed
119
120
121
122
123
	useDefaultSplit := true
	for _, s := range params.TensorSplit {
		if s != 0 {
			useDefaultSplit = false
			break
124
		}
Michael Yang's avatar
Michael Yang committed
125
	}
126

Michael Yang's avatar
Michael Yang committed
127
128
129
130
	// calculate splits
	splits := make([]float32, len(gpus))
	if useDefaultSplit {
		// default: split on free memory
131
132
133
134
135
		for i := range splits {
			var free, total C.size_t
			C.ggml_backend_dev_memory(gpus[i], &free, &total)
			splits[i] = float32(free)
		}
Michael Yang's avatar
Michael Yang committed
136
137
	} else {
		splits = params.TensorSplit
138
139
140
	}

	var sum float32
Michael Yang's avatar
Michael Yang committed
141
	// cumulative sum of all splits
142
143
144
145
146
	for i := range splits {
		sum += splits[i]
		splits[i] = sum
	}

Michael Yang's avatar
Michael Yang committed
147
	// normalize splits
148
	for i := range splits {
149
		splits[i] /= sum
150
151
	}

Michael Yang's avatar
Michael Yang committed
152
	// inputs always use cpu
Michael Yang's avatar
Michael Yang committed
153
	input := cpuDeviceBufferType
154

155
	blocks := int(meta.KV().BlockCount())
Michael Yang's avatar
Michael Yang committed
156
157
158
159

	// define a range of gpu layers. anything outside of this range is assigned to the cpu
	gpuRangeStart := max(0, blocks-params.NumGPULayers)
	gpuRangeStop := min(gpuRangeStart+params.NumGPULayers, blocks+1)
Michael Yang's avatar
Michael Yang committed
160
	assignLayer := func(i int) deviceBufferType {
Michael Yang's avatar
Michael Yang committed
161
		if i < gpuRangeStart || i >= gpuRangeStop {
Michael Yang's avatar
Michael Yang committed
162
			return cpuDeviceBufferType
163
		}
164

Michael Yang's avatar
Michael Yang committed
165
		index := slices.IndexFunc(splits, func(f float32) bool { return float32(i-gpuRangeStart)/float32(gpuRangeStop-gpuRangeStart) < f })
166
		if index < 0 || index >= len(gpuDeviceBufferTypes) {
Michael Yang's avatar
Michael Yang committed
167
			return cpuDeviceBufferType
168
169
170
		}

		return gpuDeviceBufferTypes[index]
171
172
	}

Michael Yang's avatar
Michael Yang committed
173
	// repeating layers are assigned based on their index in reverse order, e.g. i / (block_count + 1)
174
	layers := make([]deviceBufferType, blocks)
175
	for i := range layers {
176
		layers[i] = assignLayer(i)
177
178
	}

Michael Yang's avatar
Michael Yang committed
179
	// outputs are assigned iff allowed by splits and configured number of gpu layers
180
	output := assignLayer(blocks)
181
182
183

	maxTensors := len(meta.Tensors().Items())
	maxTensors += 1
Michael Yang's avatar
Michael Yang committed
184
	// each layer has at most 2 extra tensors for rope operations
185
186
	maxTensors += blocks * 2

187
188
189
190
191
	type tensor struct {
		source *fs.Tensor
		target string
	}

Michael Yang's avatar
Michael Yang committed
192
	// some tensors are mapped to different names so keep a list
193
194
	targets := make(map[string][]string)

Michael Yang's avatar
Michael Yang committed
195
	// contexts are shared by tensors of the same buffer type
196
	ctxs := make(map[*C.struct_ggml_backend_buffer_type]*C.struct_ggml_context)
197
	createTensor := func(t tensor, bts []*C.struct_ggml_backend_buffer_type) *C.struct_ggml_tensor {
198
199
200
201
202
203
204
		for _, bt := range bts {
			if _, ok := ctxs[bt]; !ok {
				ctxs[bt] = C.ggml_init(C.struct_ggml_init_params{
					mem_size: C.ggml_tensor_overhead() * C.size_t(maxTensors),
					no_alloc: true,
				})
			}
Michael Yang's avatar
Michael Yang committed
205

206
207
208
209
210
211
212
213
			targets[t.source.Name] = append(targets[t.source.Name], t.target)

			name := t.source.Name
			if t.target != "" {
				name = t.target
			}

			cname := C.CString(name)
Michael Yang's avatar
Michael Yang committed
214
			defer C.free(unsafe.Pointer(cname))
215
216
217
218
			if tt := C.ggml_get_tensor(ctxs[bt], cname); tt != nil {
				return tt
			}

219
			tt := C.ggml_new_tensor(ctxs[bt], t.source.Kind, C.int(len(t.source.Shape)), (*C.int64_t)(unsafe.Pointer(&t.source.Shape[0])))
Michael Yang's avatar
Michael Yang committed
220
221
			C.ggml_set_name(tt, cname)

222
			slog.Debug("created tensor", "name", name, "shape", t.source.Shape, "dtype", t.source.Kind, "buffer_type", C.GoString(C.ggml_backend_buft_name(bt)))
223
224
225
226
227
			//nolint:staticcheck // TODO: check if buffer type supports this tensor
			return tt
		}

		return nil
Michael Yang's avatar
Michael Yang committed
228
229
	}

230
	contains := func(s string, parts ...string) bool {
231
232
233
234
235
236
237
238
		split := strings.Split(s, ".")
		for _, part := range parts {
			if slices.Contains(split, part) {
				return true
			}
		}

		return false
Michael Yang's avatar
Michael Yang committed
239
240
	}

241
242
	for _, t := range meta.Tensors().Items() {
		switch {
243
		case contains(t.Name, "position_embd", "token_embd", "token_norm_embd", "token_types"):
244
			createTensor(tensor{source: t}, input.bts)
Michael Yang's avatar
Michael Yang committed
245
246
247
			if _, ok := meta.Tensors().GroupLayers()["output"]; !ok && t.Name == "token_embd.weight" {
				createTensor(tensor{source: t, target: "output.weight"}, output.bts)
			}
248
		case contains(t.Name, "cls", "output", "output_norm"):
249
			createTensor(tensor{source: t}, output.bts)
250
		case strings.HasPrefix(t.Name, "v.") || strings.HasPrefix(t.Name, "mm."):
Michael Yang's avatar
Michael Yang committed
251
			// TODO: assign vision tensors to the gpu if possible
Michael Yang's avatar
Michael Yang committed
252
			createTensor(tensor{source: t}, output.bts)
Michael Yang's avatar
Michael Yang committed
253
254
255
256
257
258
259
260
		case contains(t.Name, "rope_freqs", "rope_factors_long", "rope_factors_short"):
			// these tensors should be repeated per layer
			for i, layer := range layers {
				createTensor(tensor{
					source: t,
					target: "blk." + strconv.Itoa(i) + "." + t.Name,
				}, layer.bts)
			}
261
		default:
Michael Yang's avatar
Michael Yang committed
262
263
264
265
			layerIndex := -1
			if fields := strings.FieldsFunc(t.Name, func(r rune) bool { return !unicode.IsNumber(r) }); len(fields) > 0 {
				if i, err := strconv.Atoi(fields[0]); err == nil {
					layerIndex = i
266
				}
Michael Yang's avatar
Michael Yang committed
267
			}
268

Michael Yang's avatar
Michael Yang committed
269
270
			if layerIndex >= 0 {
				createTensor(tensor{source: t}, layers[layerIndex].bts)
271
			} else {
Michael Yang's avatar
Michael Yang committed
272
273
				// load all other tensors on the cpu
				createTensor(tensor{source: t}, input.bts)
274
275
276
			}
		}
	}
Michael Yang's avatar
Michael Yang committed
277

Michael Yang's avatar
Michael Yang committed
278
279
	// allocate buffers for each context
	bbs := make(map[*C.struct_ggml_context]*C.struct_ggml_backend_buffer, len(ctxs))
280
281
282
283
284
285
286
	for bt, c := range ctxs {
		if C.ggml_get_first_tensor(c) == nil {
			continue
		}

		b := C.ggml_backend_alloc_ctx_tensors_from_buft(c, bt)
		C.ggml_backend_buffer_set_usage(b, C.GGML_BACKEND_BUFFER_USAGE_WEIGHTS)
Michael Yang's avatar
Michael Yang committed
287
		bbs[c] = b
288
289
290
	}

	for bs := range maps.Values(bbs) {
Michael Yang's avatar
Michael Yang committed
291
		slog.Info("model weights", "buffer", C.GoString(C.ggml_backend_buffer_name(bs)), "size", format.HumanBytes2(uint64(C.ggml_backend_buffer_get_size(bs))))
292
293
	}

Michael Yang's avatar
Michael Yang committed
294
	// map tensor names to tensors for easy lookup later
295
296
297
298
299
300
301
	tensors := make(map[string]*C.struct_ggml_tensor)
	for _, c := range ctxs {
		for t := C.ggml_get_first_tensor(c); t != nil; t = C.ggml_get_next_tensor(c, t) {
			tensors[C.GoString(C.ggml_get_name(t))] = t
		}
	}

302
303
304
305
306
	var doneBytes atomic.Uint64
	totalBytes := uint64(n) - meta.Tensors().Offset

	g, ctx := errgroup.WithContext(ctx)
	g.SetLimit(runtime.GOMAXPROCS(0))
307
	for _, t := range meta.Tensors().Items() {
308
309
310
311
		g.Go(func() error {
			tts := make([]*C.struct_ggml_tensor, max(1, len(targets[t.Name])))
			for i := range tts {
				target := targets[t.Name][i]
312
313
314
				if target == "" {
					target = t.Name
				}
315

316
317
318
319
				tt, ok := tensors[target]
				if !ok {
					return fmt.Errorf("unassigned tensor: %s", t.Name)
				}
Michael Yang's avatar
Michael Yang committed
320

321
322
323
324
325
326
327
328
329
330
331
				tts[i] = tt
			}

			sr := io.NewSectionReader(r, int64(meta.Tensors().Offset+t.Offset), int64(t.Size()))
			bts := make([]byte, 128*format.KibiByte)

			var s uint64
			for s < t.Size() {
				n, err := io.ReadFull(sr, bts[:min(len(bts), int(t.Size()-s))])
				if err != nil {
					return err
332
				}
Michael Yang's avatar
Michael Yang committed
333

334
335
				for _, tt := range tts {
					C.ggml_backend_tensor_set(tt, unsafe.Pointer(&bts[0]), C.size_t(s), C.size_t(n))
336
				}
Michael Yang's avatar
Michael Yang committed
337

338
339
340
341
342
343
344
345
346
347
				s += uint64(n)

				if params.Progress != nil {
					done := doneBytes.Add(uint64(n))
					params.Progress(float32(done) / float32(totalBytes))
				}
			}

			return nil
		})
Michael Yang's avatar
Michael Yang committed
348
349
	}

350
351
352
353
354
355
356
357
	// start a goroutine to cancel the errgroup if the parent context is done
	go func() {
		<-ctx.Done()
		g.Go(func() error {
			return ctx.Err()
		})
	}()

358
	if err := g.Wait(); err != nil {
Michael Yang's avatar
Michael Yang committed
359
360
361
		return nil, err
	}

362
363
	// map devices to backend buffer types so new tensors can be assigned to the correct device
	deviceBufferTypes := make(map[*C.struct_ggml_backend_device]*C.struct_ggml_backend_buffer_type)
Michael Yang's avatar
Michael Yang committed
364
365
366
367

	// create backends and buffer types used for the compute graph scheduler
	var schedBackends []*C.struct_ggml_backend
	var schedBufts []*C.struct_ggml_backend_buffer_type
368
369
370
371
	for _, d := range append(gpus, append(accels, cpus...)...) {
		b := C.ggml_backend_dev_init(d, nil)
		bt := C.ggml_backend_get_default_buffer_type(b)
		if d := C.ggml_backend_get_device(b); C.ggml_backend_dev_type(d) == C.GGML_BACKEND_DEVICE_TYPE_CPU && len(gpus) > 0 {
372
373
			// use the first gpu host buffer type for gpu if possible
			if hbt := C.ggml_backend_dev_host_buffer_type(gpus[0]); hbt != nil {
374
375
376
377
				bt = hbt
			}
		}

378
379
380
		deviceBufferTypes[d] = bt

		schedBackends = append(schedBackends, b)
Michael Yang's avatar
Michael Yang committed
381
		schedBufts = append(schedBufts, bt)
382

383
		slog.Info("compute graph", "backend", C.GoString(C.ggml_backend_name(b)), "buffer_type", C.GoString(C.ggml_backend_buft_name(bt)))
384
385

		if C.ggml_backend_is_cpu(b) {
Michael Yang's avatar
Michael Yang committed
386
			// set number of threads for cpu backend
Michael Yang's avatar
Michael Yang committed
387
			C.ggml_backend_cpu_set_n_threads(b, C.int(Threads(params.NumThreads)))
388
		}
389
390
	}

Michael Yang's avatar
Michael Yang committed
391
	maxGraphNodes := max(8192, len(meta.Tensors().Items())*5)
Michael Yang's avatar
Michael Yang committed
392
	return &Backend{
393
		flashAttention: params.FlashAttention,
394
395
		meta:           meta,
		tensors:        tensors,
396
		sched: C.ggml_backend_sched_new(
Michael Yang's avatar
Michael Yang committed
397
398
399
400
			(*C.ggml_backend_t)(unsafe.Pointer(&schedBackends[0])),
			(*C.ggml_backend_buffer_type_t)(unsafe.Pointer(&schedBufts[0])),
			C.int(len(schedBackends)),
			C.size_t(maxGraphNodes),
401
			C._Bool(len(gpus) > 1 && slices.Contains(gpus, output.d)),
402
		),
403
404
405
406
		input:  deviceBufferTypes[input.d],
		output: deviceBufferTypes[output.d],
		layers: func() map[int]*C.struct_ggml_backend_buffer_type {
			m := make(map[int]*C.struct_ggml_backend_buffer_type)
407
			for i, layer := range layers {
408
				m[i] = deviceBufferTypes[layer.d]
409
410
411
			}
			return m
		}(),
Michael Yang's avatar
Michael Yang committed
412
		maxGraphNodes: maxGraphNodes,
Michael Yang's avatar
Michael Yang committed
413
414
415
416
417
418
419
420
421
422
423
424
	}, nil
}

func init() {
	ml.RegisterBackend("ggml", New)
}

func (b *Backend) Config() ml.Config {
	return b.meta.KV()
}

func (b *Backend) Get(name string) ml.Tensor {
425
426
	if t, ok := b.tensors[name]; ok {
		return &Tensor{b: b, t: t}
Michael Yang's avatar
Michael Yang committed
427
428
429
430
431
432
	}

	return nil
}

func (b *Backend) NewContext() ml.Context {
Michael Yang's avatar
Michael Yang committed
433
	return b.NewContextSize(b.maxGraphNodes)
434
435
436
}

func (b *Backend) NewContextSize(n int) ml.Context {
Jesse Gross's avatar
Jesse Gross committed
437
438
439
440
	if n > b.maxGraphNodes {
		panic(fmt.Errorf("requested number of graph nodes (%v) for new context exceeds maximum (%v)", n, b.maxGraphNodes))
	}

Michael Yang's avatar
Michael Yang committed
441
	return &Context{
442
443
		b:             b,
		maxGraphNodes: n,
444
		ctx: C.ggml_init(C.struct_ggml_init_params{
445
			mem_size: C.size_t(n)*C.ggml_tensor_overhead() + C.ggml_graph_overhead_custom(C.size_t(n), false),
446
447
			no_alloc: true,
		}),
Michael Yang's avatar
Michael Yang committed
448
449
450
	}
}

451
func (b *Backend) CacheConfig() ml.CacheConfig {
452
453
454
455
456
	if b.flashAttention {
		return ml.CacheConfig{CachePadding: 256, MaskDType: ml.DTypeF16, MaskBatchPadding: C.GGML_KQ_MASK_PAD}
	} else {
		return ml.CacheConfig{CachePadding: 32, PermutedV: true}
	}
457
458
}

Michael Yang's avatar
Michael Yang committed
459
type Context struct {
460
	b *Backend
Michael Yang's avatar
Michael Yang committed
461

462
	ctx   *C.struct_ggml_context
Michael Yang's avatar
Michael Yang committed
463
	graph *C.struct_ggml_cgraph
464

465
466
	// buft is the buffer type used for new tensors
	buft *C.struct_ggml_backend_buffer_type
467

Michael Yang's avatar
Michael Yang committed
468
	// maxGraphNodes is the maximum allowed number of graph nodes in this context
469
	maxGraphNodes int
Michael Yang's avatar
Michael Yang committed
470
471
}

Michael Yang's avatar
Michael Yang committed
472
473
func (c Context) Input() ml.Context {
	if c.b.input != nil {
474
475
476
		return &Context{
			b:             c.b,
			ctx:           c.ctx,
477
			buft:          c.b.input,
478
479
480
481
			maxGraphNodes: c.maxGraphNodes,
		}
	}

Michael Yang's avatar
Michael Yang committed
482
	return &c
483
484
}

Michael Yang's avatar
Michael Yang committed
485
486
func (c Context) Output() ml.Context {
	if c.b.output != nil {
487
488
489
		return &Context{
			b:             c.b,
			ctx:           c.ctx,
490
			buft:          c.b.output,
491
492
493
494
			maxGraphNodes: c.maxGraphNodes,
		}
	}

Michael Yang's avatar
Michael Yang committed
495
	return &c
496
497
}

Michael Yang's avatar
Michael Yang committed
498
func (c Context) Layer(i int) ml.Context {
499
	if buft, ok := c.b.layers[i]; ok {
500
501
502
		return &Context{
			b:             c.b,
			ctx:           c.ctx,
503
			buft:          buft,
504
505
506
507
			maxGraphNodes: c.maxGraphNodes,
		}
	}

Michael Yang's avatar
Michael Yang committed
508
	return &c
509
510
}

511
func (c *Context) Forward(tensors ...ml.Tensor) ml.Context {
Michael Yang's avatar
Michael Yang committed
512
	if c.graph == nil {
513
		c.graph = C.ggml_new_graph_custom(c.ctx, C.size_t(c.maxGraphNodes), false)
Michael Yang's avatar
Michael Yang committed
514
515
	}

516
517
518
519
520
	for _, tensor := range tensors {
		C.ggml_build_forward_expand(c.graph, tensor.(*Tensor).t)
	}

	return c
Michael Yang's avatar
Michael Yang committed
521
522
}

Michael Yang's avatar
Michael Yang committed
523
func (c Context) Compute(tensors ...ml.Tensor) {
524
	C.ggml_backend_sched_graph_compute_async(c.b.sched, c.graph)
Michael Yang's avatar
Michael Yang committed
525
	C.ggml_backend_sched_reset(c.b.sched)
Michael Yang's avatar
Michael Yang committed
526

527
528
529
	needSync := true
	sync := func() {
		if needSync {
530
			C.ggml_backend_sched_synchronize(c.b.sched)
531
532
533
			needSync = false
		}
	}
Michael Yang's avatar
Michael Yang committed
534

535
536
537
	for _, t := range tensors {
		if C.ggml_nbytes(t.(*Tensor).t) > 0 {
			t.(*Tensor).sync = sync
538
539
		}
	}
Michael Yang's avatar
Michael Yang committed
540
541
}

Michael Yang's avatar
Michael Yang committed
542
func (c Context) MaxGraphNodes() int {
543
	return c.maxGraphNodes
Jesse Gross's avatar
Jesse Gross committed
544
545
}

546
547
548
func shapeToGGML(shape []int) *C.int64_t {
	sh := make([]C.int64_t, len(shape))
	for i, s := range shape {
549
		sh[i] = C.int64_t(s)
550
551
552
553
554
	}

	return &sh[0]
}

555
556
557
558
func pad(length, pad C.size_t) C.size_t {
	return ((length + pad - 1) / pad) * pad
}

559
func (c Context) newTensor(dtype ml.DType, shape []int) ml.Tensor {
560
561
562
563
	if c.buft == nil {
		panic("set Input, Output, or Layer before creating tensors")
	}

Michael Yang's avatar
Michael Yang committed
564
565
566
567
568
569
	var cdtype uint32
	switch dtype {
	case ml.DTypeF32:
		cdtype = C.GGML_TYPE_F32
	case ml.DTypeF16:
		cdtype = C.GGML_TYPE_F16
570
571
572
573
	case ml.DTypeQ80:
		cdtype = C.GGML_TYPE_Q8_0
	case ml.DTypeQ40:
		cdtype = C.GGML_TYPE_Q4_0
Michael Yang's avatar
Michael Yang committed
574
575
576
577
578
579
	case ml.DTypeI32:
		cdtype = C.GGML_TYPE_I32
	default:
		panic("unsupported dtype")
	}

Jesse Gross's avatar
Jesse Gross committed
580
	if len(shape) < 1 || shape[0] == 0 {
Michael Yang's avatar
Michael Yang committed
581
582
583
		var shape C.int64_t = 0
		return &Tensor{b: c.b, t: C.ggml_new_tensor(c.ctx, cdtype, 1, &shape)}
	} else if len(shape) > 4 {
Michael Yang's avatar
Michael Yang committed
584
585
586
587
588
589
590
591
592
		panic("unsupported number of dimensions")
	}

	for _, dim := range shape {
		if dim < 1 {
			panic("invalid shape")
		}
	}

Michael Yang's avatar
Michael Yang committed
593
	t := C.ggml_new_tensor(c.ctx, cdtype, C.int(len(shape)), shapeToGGML(shape))
594
595
	size := pad(C.ggml_backend_buft_get_alloc_size(c.buft, t), C.ggml_backend_buft_get_alignment(c.buft))
	b := C.ggml_backend_buft_alloc_buffer(c.buft, size)
Michael Yang's avatar
Michael Yang committed
596
	C.ggml_backend_tensor_alloc(b, t, C.ggml_backend_buffer_get_base(b))
597
	return &Tensor{b: c.b, t: t}
598
599
600
}

func (c Context) Empty(dtype ml.DType, shape ...int) ml.Tensor {
601
	return c.newTensor(dtype, shape)
602
603
604
}

func (c Context) Zeros(dtype ml.DType, shape ...int) ml.Tensor {
605
	t := c.newTensor(dtype, shape)
606
607
	C.ggml_set_zero(t.(*Tensor).t)
	return t
Michael Yang's avatar
Michael Yang committed
608
609
}

610
func checkShape[S ~[]E, E any](s S, shape ...int) error {
Michael Yang's avatar
Michael Yang committed
611
	n := len(s)
Jesse Gross's avatar
Jesse Gross committed
612
613
614
615
616

	if n == 0 {
		return nil
	}

Michael Yang's avatar
Michael Yang committed
617
618
619
620
621
	for _, v := range shape {
		n /= v
	}

	if n != 1 {
622
		return fmt.Errorf("invalid shape: %v", shape)
Michael Yang's avatar
Michael Yang committed
623
624
	}

625
	return nil
Michael Yang's avatar
Michael Yang committed
626
627
628
}

func (c Context) FromFloatSlice(s []float32, shape ...int) (ml.Tensor, error) {
Jesse Gross's avatar
Jesse Gross committed
629
	if err := checkShape(s, shape...); err != nil {
630
631
632
633
		return nil, err
	}

	t := c.newTensor(ml.DTypeF32, shape)
Jesse Gross's avatar
Jesse Gross committed
634
635
636
637
	if len(s) > 0 {
		C.ggml_backend_tensor_set(t.(*Tensor).t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t.(*Tensor).t))
	}

638
	return t, nil
Michael Yang's avatar
Michael Yang committed
639
640
641
}

func (c Context) FromIntSlice(s []int32, shape ...int) (ml.Tensor, error) {
Jesse Gross's avatar
Jesse Gross committed
642
	if err := checkShape(s, shape...); err != nil {
643
644
645
646
		return nil, err
	}

	t := c.newTensor(ml.DTypeI32, shape)
Jesse Gross's avatar
Jesse Gross committed
647
648
649
650
	if len(s) > 0 {
		C.ggml_backend_tensor_set(t.(*Tensor).t, unsafe.Pointer(&s[0]), 0, C.ggml_nbytes(t.(*Tensor).t))
	}

651
	return t, nil
Michael Yang's avatar
Michael Yang committed
652
653
}

Michael Yang's avatar
Michael Yang committed
654
655
func (c *Context) Close() {
	if c != nil {
656
657
		C.ggml_free(c.ctx)
	}
Michael Yang's avatar
Michael Yang committed
658
659
660
}

type Tensor struct {
661
	b    *Backend
Michael Yang's avatar
Michael Yang committed
662
	t    *C.struct_ggml_tensor
663
	sync func()
Michael Yang's avatar
Michael Yang committed
664
665
666
667
668
669
670
671
672
673
}

func (t *Tensor) LogValue() slog.Value {
	return slog.GroupValue(
		slog.String("name", C.GoString(C.ggml_get_name(t.t))),
		slog.String("type", C.GoString(C.ggml_type_name(t.t._type))),
		slog.Any("shape", t.Shape()),
	)
}

674
675
func (t *Tensor) Dim(n int) int {
	return int(t.t.ne[n])
Michael Yang's avatar
Michael Yang committed
676
677
}

678
679
func (t *Tensor) Stride(n int) int {
	return int(t.t.nb[n])
Michael Yang's avatar
Michael Yang committed
680
681
}

682
683
func (t *Tensor) Shape() []int {
	shape := make([]int, C.ggml_n_dims(t.t))
Michael Yang's avatar
Michael Yang committed
684
685
686
687
688
689
690
	for i := range shape {
		shape[i] = t.Dim(i)
	}

	return shape
}

691
692
693
694
695
696
697
698
699
func (t *Tensor) Bytes() (data []byte) {
	if t.sync != nil {
		data = make([]byte, C.ggml_nbytes(t.t))

		t.sync()
		C.ggml_backend_tensor_get(t.t, unsafe.Pointer(&data[0]), 0, C.ggml_nbytes(t.t))
	}

	return
Michael Yang's avatar
Michael Yang committed
700
701
}

702
703
704
705
706
707
func (t *Tensor) Floats() (data []float32) {
	if t.sync != nil {
		data = make([]float32, C.ggml_nelements(t.t))

		t.sync()
		C.ggml_backend_tensor_get(t.t, unsafe.Pointer(&data[0]), 0, C.ggml_nbytes(t.t))
Michael Yang's avatar
Michael Yang committed
708
709
710
711
712
713
714
715
716
	}

	return
}

func (t *Tensor) DType() ml.DType {
	switch t.t._type {
	case C.GGML_TYPE_F32:
		return ml.DTypeF32
Jesse Gross's avatar
Jesse Gross committed
717
718
	case C.GGML_TYPE_F16:
		return ml.DTypeF16
719
720
721
722
	case C.GGML_TYPE_Q8_0:
		return ml.DTypeQ80
	case C.GGML_TYPE_Q4_0:
		return ml.DTypeQ40
Michael Yang's avatar
Michael Yang committed
723
724
725
726
727
728
729
730
731
	case C.GGML_TYPE_I32:
		return ml.DTypeI32
	default:
		return ml.DTypeOther
	}
}

func (t *Tensor) Add(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
	return &Tensor{
732
		b: t.b,
Michael Yang's avatar
Michael Yang committed
733
734
735
736
737
738
739
740
741
742
743
744
745
746
		t: C.ggml_add(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
	}
}

func (t *Tensor) Stack(ctx ml.Context, dim int, s ...ml.Tensor) ml.Tensor {
	if len(s) > 0 {
		return t.Concat(ctx, s[0].Stack(ctx, dim, s[1:]...), dim)
	}

	return t
}

func (t *Tensor) Concat(ctx ml.Context, t2 ml.Tensor, dim int) ml.Tensor {
	return &Tensor{
747
		b: t.b,
Michael Yang's avatar
Michael Yang committed
748
749
750
751
752
753
		t: C.ggml_concat(ctx.(*Context).ctx, t.t, t2.(*Tensor).t, C.int(dim)),
	}
}

func (t *Tensor) Contiguous(ctx ml.Context) ml.Tensor {
	return &Tensor{
754
		b: t.b,
Michael Yang's avatar
Michael Yang committed
755
756
757
758
759
760
		t: C.ggml_cont(ctx.(*Context).ctx, t.t),
	}
}

func (t *Tensor) Mul(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
	return &Tensor{
761
		b: t.b,
Michael Yang's avatar
Michael Yang committed
762
763
764
765
766
767
		t: C.ggml_mul(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
	}
}

func (t *Tensor) Mulmat(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
	return &Tensor{
768
		b: t.b,
Michael Yang's avatar
Michael Yang committed
769
770
771
772
		t: C.ggml_mul_mat(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
	}
}

773
774
775
776
777
func (t *Tensor) MulmatFullPrec(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
	mul := C.ggml_mul_mat(ctx.(*Context).ctx, t.t, t2.(*Tensor).t)
	C.ggml_mul_mat_set_prec(mul, C.GGML_PREC_F32)

	return &Tensor{
778
		b: t.b,
779
780
781
782
		t: mul,
	}
}

Michael Yang's avatar
Michael Yang committed
783
func (t *Tensor) LayerNorm(ctx ml.Context, w, b ml.Tensor, eps float32) ml.Tensor {
784
	tt := (&Tensor{b: t.b, t: C.ggml_norm(ctx.(*Context).ctx, t.t, C.float(eps))}).Mul(ctx, w)
Michael Yang's avatar
Michael Yang committed
785
786
787
788
789
790
791
792
	if b != nil {
		tt = tt.Add(ctx, b)
	}

	return tt
}

func (t *Tensor) RMSNorm(ctx ml.Context, w ml.Tensor, eps float32) ml.Tensor {
793
	return (&Tensor{b: t.b, t: C.ggml_rms_norm(ctx.(*Context).ctx, t.t, C.float(eps))}).Mul(ctx, w)
Michael Yang's avatar
Michael Yang committed
794
795
}

796
func (t *Tensor) Pad(ctx ml.Context, shape ...int) ml.Tensor {
Michael Yang's avatar
Michael Yang committed
797
798
799
800
801
	if len(shape) != 4 {
		panic("expected 4 dimensions")
	}

	return &Tensor{
802
		b: t.b,
Michael Yang's avatar
Michael Yang committed
803
804
805
806
807
808
809
810
811
812
		t: C.ggml_pad(ctx.(*Context).ctx, t.t, C.int(shape[0]), C.int(shape[1]), C.int(shape[2]), C.int(shape[3])),
	}
}

func (t *Tensor) Permute(ctx ml.Context, shape ...int) ml.Tensor {
	if len(shape) != 4 {
		panic("expected 4 dimensions")
	}

	return &Tensor{
813
		b: t.b,
Michael Yang's avatar
Michael Yang committed
814
815
816
817
818
819
		t: C.ggml_permute(ctx.(*Context).ctx, t.t, C.int(shape[0]), C.int(shape[1]), C.int(shape[2]), C.int(shape[3])),
	}
}

func (t *Tensor) Rows(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
	return &Tensor{
820
		b: t.b,
Michael Yang's avatar
Michael Yang committed
821
822
823
824
825
826
		t: C.ggml_get_rows(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
	}
}

func (t *Tensor) Copy(ctx ml.Context, t2 ml.Tensor) ml.Tensor {
	return &Tensor{
827
		b: t.b,
Michael Yang's avatar
Michael Yang committed
828
829
830
831
		t: C.ggml_cpy(ctx.(*Context).ctx, t.t, t2.(*Tensor).t),
	}
}

832
func (t *Tensor) Reshape(ctx ml.Context, shape ...int) ml.Tensor {
Michael Yang's avatar
Michael Yang committed
833
834
835
	switch len(shape) {
	case 1:
		return &Tensor{
836
			b: t.b,
Michael Yang's avatar
Michael Yang committed
837
838
839
840
			t: C.ggml_reshape_1d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0])),
		}
	case 2:
		return &Tensor{
841
			b: t.b,
Michael Yang's avatar
Michael Yang committed
842
843
844
845
			t: C.ggml_reshape_2d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.int64_t(shape[1])),
		}
	case 3:
		return &Tensor{
846
			b: t.b,
Michael Yang's avatar
Michael Yang committed
847
848
849
850
			t: C.ggml_reshape_3d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.int64_t(shape[1]), C.int64_t(shape[2])),
		}
	case 4:
		return &Tensor{
851
			b: t.b,
Michael Yang's avatar
Michael Yang committed
852
853
854
855
856
857
858
859
860
			t: C.ggml_reshape_4d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.int64_t(shape[1]), C.int64_t(shape[2]), C.int64_t(shape[3])),
		}
	default:
		panic("unsupported number of dimensions")
	}
}

func (t *Tensor) Scale(ctx ml.Context, s float64) ml.Tensor {
	return &Tensor{
861
		b: t.b,
Michael Yang's avatar
Michael Yang committed
862
863
864
865
866
867
		t: C.ggml_scale(ctx.(*Context).ctx, t.t, (C.float)(s)),
	}
}

func (t *Tensor) Softmax(ctx ml.Context) ml.Tensor {
	return &Tensor{
868
		b: t.b,
Michael Yang's avatar
Michael Yang committed
869
870
871
872
873
874
		t: C.ggml_soft_max(ctx.(*Context).ctx, t.t),
	}
}

func (t *Tensor) Tanh(ctx ml.Context) ml.Tensor {
	return &Tensor{
875
		b: t.b,
Michael Yang's avatar
Michael Yang committed
876
877
878
879
		t: C.ggml_tanh_inplace(ctx.(*Context).ctx, t.t),
	}
}

880
func (t *Tensor) Unpad(ctx ml.Context, shape ...int) ml.Tensor {
Michael Yang's avatar
Michael Yang committed
881
882
883
884
885
	if len(shape) != 4 {
		panic("expected 4 dimensions")
	}

	return &Tensor{
886
		b: t.b,
Michael Yang's avatar
Michael Yang committed
887
888
889
890
891
892
893
894
		t: C.ggml_unpad(ctx.(*Context).ctx, t.t, C.int(shape[0]), C.int(shape[1]), C.int(shape[2]), C.int(shape[3])),
	}
}

func (t *Tensor) View(ctx ml.Context, offset int, shape ...int) ml.Tensor {
	switch len(shape) {
	case 1:
		return &Tensor{
895
			b: t.b,
Michael Yang's avatar
Michael Yang committed
896
897
898
899
			t: C.ggml_view_1d(ctx.(*Context).ctx, t.t, C.int64_t(shape[0]), C.size_t(offset)),
		}
	case 3:
		return &Tensor{
900
			b: t.b,
Michael Yang's avatar
Michael Yang committed
901
902
903
904
905
906
907
			t: C.ggml_view_2d(ctx.(*Context).ctx, t.t,
				C.int64_t(shape[0]), C.int64_t(shape[2]),
				C.size_t(shape[1]),
				C.size_t(offset)),
		}
	case 5:
		return &Tensor{
908
			b: t.b,
Michael Yang's avatar
Michael Yang committed
909
910
911
912
913
914
915
			t: C.ggml_view_3d(ctx.(*Context).ctx, t.t,
				C.int64_t(shape[0]), C.int64_t(shape[2]), C.int64_t(shape[4]),
				C.size_t(shape[1]), C.size_t(shape[3]),
				C.size_t(offset)),
		}
	case 7:
		return &Tensor{
916
			b: t.b,
Michael Yang's avatar
Michael Yang committed
917
918
919
920
921
922
923
924
925
926
927
			t: C.ggml_view_4d(ctx.(*Context).ctx, t.t,
				C.int64_t(shape[0]), C.int64_t(shape[2]), C.int64_t(shape[4]), C.int64_t(shape[6]),
				C.size_t(shape[1]), C.size_t(shape[3]), C.size_t(shape[5]),
				C.size_t(offset)),
		}
	default:
		panic("unsupported number of dimensions")
	}
}

const (
Patrick Devine's avatar
Patrick Devine committed
928
929
930
931
	ropeTypeNorm   C.int = 0
	ropeTypeNeox   C.int = 2
	ropeTypeMrope  C.int = 8
	ropeTypeVision C.int = 24
Michael Yang's avatar
Michael Yang committed
932
933
)

Patrick Devine's avatar
Patrick Devine committed
934
func (t *Tensor) RoPE(ctx ml.Context, positionIDs, ropeFactors ml.Tensor, ropeDim, ropeType uint32, ropeBase, ropeScale float32) ml.Tensor {
Michael Yang's avatar
Michael Yang committed
935
	if ropeFactors == nil {
936
		ropeFactors = &Tensor{b: t.b}
Michael Yang's avatar
Michael Yang committed
937
938
	}

Jesse Gross's avatar
Jesse Gross committed
939
940
941
942
943
	dequant := t.t
	if C.ggml_is_quantized(t.t._type) {
		dequant = C.ggml_cast(ctx.(*Context).ctx, t.t, C.GGML_TYPE_F32)
	}

Michael Yang's avatar
Michael Yang committed
944
	return &Tensor{
945
		b: t.b,
Michael Yang's avatar
Michael Yang committed
946
		t: C.ggml_rope_ext(
Jesse Gross's avatar
Jesse Gross committed
947
			ctx.(*Context).ctx, dequant, positionIDs.(*Tensor).t, ropeFactors.(*Tensor).t,
Michael Yang's avatar
Michael Yang committed
948
			C.int(ropeDim),
Patrick Devine's avatar
Patrick Devine committed
949
950
			C.int(ropeType),
			131072, // YaRN n_ctx_train
Michael Yang's avatar
Michael Yang committed
951
952
953
954
955
956
957
958
959
960
961
962
			C.float(ropeBase),
			C.float(ropeScale),
			0.,  // YaRN ext_factor
			1.,  // YaRN attn_factor
			32., // YaRN beta_fast
			1.,  // YaRN beta_slow
		),
	}
}

func (t *Tensor) GELU(ctx ml.Context) ml.Tensor {
	return &Tensor{
963
		b: t.b,
Michael Yang's avatar
Michael Yang committed
964
965
966
967
968
969
		t: C.ggml_gelu_inplace(ctx.(*Context).ctx, t.t),
	}
}

func (t *Tensor) SILU(ctx ml.Context) ml.Tensor {
	return &Tensor{
970
		b: t.b,
Michael Yang's avatar
Michael Yang committed
971
972
973
974
975
976
		t: C.ggml_silu_inplace(ctx.(*Context).ctx, t.t),
	}
}

func (t *Tensor) Conv2D(ctx ml.Context, t2 ml.Tensor, s0, s1, p0, p1, d0, d1 int) ml.Tensor {
	return &Tensor{
977
		b: t.b,
Michael Yang's avatar
Michael Yang committed
978
979
980
		t: C.ggml_conv_2d(ctx.(*Context).ctx, t.t, t2.(*Tensor).t, C.int(s0), C.int(s1), C.int(p0), C.int(p1), C.int(d0), C.int(d1)),
	}
}
981

Michael Yang's avatar
Michael Yang committed
982
func (t *Tensor) AvgPool2D(ctx ml.Context, k, s int, p float32) ml.Tensor {
Michael Yang's avatar
Michael Yang committed
983
984
	return &Tensor{
		b: t.b,
Michael Yang's avatar
Michael Yang committed
985
		t: C.ggml_pool_2d(ctx.(*Context).ctx, t.t, C.GGML_OP_POOL_AVG, C.int(k), C.int(k), C.int(s), C.int(s), C.float(p), C.float(p)),
Michael Yang's avatar
Michael Yang committed
986
987
988
	}
}

Michael Yang's avatar
Michael Yang committed
989
990
991
992
func (t *Tensor) Set(ctx ml.Context, t2 ml.Tensor, offset int, strides ...int) ml.Tensor {
	var tt *C.struct_ggml_tensor
	switch len(strides) {
	case 0:
Michael Yang's avatar
Michael Yang committed
993
		tt = C.ggml_set_1d(ctx.(*Context).ctx, t.t, t2.(*Tensor).t, C.size_t(offset))
Michael Yang's avatar
Michael Yang committed
994
	case 1:
Michael Yang's avatar
Michael Yang committed
995
		tt = C.ggml_set_2d(ctx.(*Context).ctx, t.t, t2.(*Tensor).t, C.size_t(offset), C.size_t(strides[0]))
Michael Yang's avatar
Michael Yang committed
996
997
998
999
1000
1001
1002
	default:
		panic("unsupported number of dimensions")
	}

	return &Tensor{b: t.b, t: tt}
}

1003
1004
1005
1006
1007
1008
func (t *Tensor) ScaledDotProductAttention(ctx ml.Context, key, value, mask ml.Tensor, scale float64) ml.Tensor {
	var kqMask *C.struct_ggml_tensor
	if mask != nil {
		kqMask = mask.(*Tensor).t
	}

1009
1010
1011
	query := t.Permute(ctx, 0, 2, 1, 3)
	key = key.Permute(ctx, 0, 2, 1, 3)

1012
1013
	if t.b.flashAttention {
		value = value.Permute(ctx, 0, 2, 1, 3)
1014

1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
		kqv := C.ggml_flash_attn_ext(ctx.(*Context).ctx, query.(*Tensor).t, key.(*Tensor).t, value.(*Tensor).t, kqMask, C.float(scale), 0, 0)
		C.ggml_flash_attn_ext_set_prec(kqv, C.GGML_PREC_F32)
		return &Tensor{b: t.b, t: kqv}
	} else {
		kq := key.MulmatFullPrec(ctx, query)
		kq = &Tensor{
			b: t.b,
			t: C.ggml_soft_max_ext(ctx.(*Context).ctx, kq.(*Tensor).t, kqMask, C.float(scale), 0),
		}

		kqv := value.Mulmat(ctx, kq)
		return kqv.Permute(ctx, 0, 2, 1, 3).Contiguous(ctx)
	}
1028
}