images.go 27.2 KB
Newer Older
1
2
3
4
package server

import (
	"bytes"
5
	"context"
6
	"crypto/sha256"
Patrick Devine's avatar
Patrick Devine committed
7
	"encoding/hex"
8
9
10
11
12
13
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
Michael Yang's avatar
Michael Yang committed
14
	"net/url"
15
16
	"os"
	"path/filepath"
Michael Yang's avatar
Michael Yang committed
17
	"runtime"
18
19
	"strconv"
	"strings"
Quinn Slack's avatar
Quinn Slack committed
20
	"text/template"
21

Michael Yang's avatar
Michael Yang committed
22
23
	"golang.org/x/exp/slices"

24
	"github.com/jmorganca/ollama/api"
25
	"github.com/jmorganca/ollama/llm"
26
	"github.com/jmorganca/ollama/parser"
Michael Yang's avatar
Michael Yang committed
27
	"github.com/jmorganca/ollama/version"
28
29
)

30
31
32
33
type RegistryOptions struct {
	Insecure bool
	Username string
	Password string
Patrick Devine's avatar
Patrick Devine committed
34
	Token    string
35
36
}

37
type Model struct {
Patrick Devine's avatar
Patrick Devine committed
38
39
40
41
42
43
44
45
46
47
	Name          string `json:"name"`
	ShortName     string
	ModelPath     string
	OriginalModel string
	AdapterPaths  []string
	Template      string
	System        string
	License       []string
	Digest        string
	Options       map[string]interface{}
48
49
}

50
func (m *Model) Prompt(request api.GenerateRequest) (string, error) {
51
52
53
54
55
56
	t := m.Template
	if request.Template != "" {
		t = request.Template
	}

	tmpl, err := template.New("").Parse(t)
57
58
59
60
61
	if err != nil {
		return "", err
	}

	var vars struct {
Michael Yang's avatar
Michael Yang committed
62
		First  bool
63
64
65
66
		System string
		Prompt string
	}

Michael Yang's avatar
Michael Yang committed
67
	vars.First = len(request.Context) == 0
68
	vars.System = m.System
69
70
	vars.Prompt = request.Prompt

71
72
73
74
	if request.System != "" {
		vars.System = request.System
	}

75
76
77
78
79
80
81
82
	var sb strings.Builder
	if err := tmpl.Execute(&sb, vars); err != nil {
		return "", err
	}

	return sb.String(), nil
}

83
84
85
86
87
88
89
90
91
92
type ManifestV2 struct {
	SchemaVersion int      `json:"schemaVersion"`
	MediaType     string   `json:"mediaType"`
	Config        Layer    `json:"config"`
	Layers        []*Layer `json:"layers"`
}

type Layer struct {
	MediaType string `json:"mediaType"`
	Digest    string `json:"digest"`
Michael Yang's avatar
Michael Yang committed
93
	Size      int64  `json:"size"`
Michael Yang's avatar
Michael Yang committed
94
	From      string `json:"from,omitempty"`
95
96
}

Michael Yang's avatar
Michael Yang committed
97
type LayerReader struct {
98
	Layer
Michael Yang's avatar
Michael Yang committed
99
	io.Reader
100
101
102
}

type ConfigV2 struct {
Michael Yang's avatar
Michael Yang committed
103
104
105
106
107
	ModelFormat string `json:"model_format"`
	ModelFamily string `json:"model_family"`
	ModelType   string `json:"model_type"`
	FileType    string `json:"file_type"`
	RootFS      RootFS `json:"rootfs"`
108
109

	// required by spec
110
111
112
113
114
115
116
117
118
	Architecture string `json:"architecture"`
	OS           string `json:"os"`
}

type RootFS struct {
	Type    string   `json:"type"`
	DiffIDs []string `json:"diff_ids"`
}

Michael Yang's avatar
Michael Yang committed
119
func (m *ManifestV2) GetTotalSize() (total int64) {
Patrick Devine's avatar
Patrick Devine committed
120
121
122
	for _, layer := range m.Layers {
		total += layer.Size
	}
Michael Yang's avatar
Michael Yang committed
123

Patrick Devine's avatar
Patrick Devine committed
124
125
126
127
	total += m.Config.Size
	return total
}

Patrick Devine's avatar
Patrick Devine committed
128
func GetManifest(mp ModelPath) (*ManifestV2, string, error) {
129
	fp, err := mp.GetManifestPath()
130
	if err != nil {
Patrick Devine's avatar
Patrick Devine committed
131
		return nil, "", err
132
	}
133

134
	if _, err = os.Stat(fp); err != nil {
Patrick Devine's avatar
Patrick Devine committed
135
		return nil, "", err
136
137
138
139
	}

	var manifest *ManifestV2

140
	bts, err := os.ReadFile(fp)
141
	if err != nil {
Patrick Devine's avatar
Patrick Devine committed
142
		return nil, "", fmt.Errorf("couldn't open file '%s'", fp)
143
144
	}

Patrick Devine's avatar
Patrick Devine committed
145
146
147
	shaSum := sha256.Sum256(bts)
	shaStr := hex.EncodeToString(shaSum[:])

148
	if err := json.Unmarshal(bts, &manifest); err != nil {
Patrick Devine's avatar
Patrick Devine committed
149
		return nil, "", err
150
151
	}

Patrick Devine's avatar
Patrick Devine committed
152
	return manifest, shaStr, nil
153
154
155
}

func GetModel(name string) (*Model, error) {
156
	mp := ParseModelPath(name)
Patrick Devine's avatar
Patrick Devine committed
157
	manifest, digest, err := GetManifest(mp)
158
159
160
161
162
	if err != nil {
		return nil, err
	}

	model := &Model{
163
164
165
166
167
		Name:      mp.GetFullTagname(),
		ShortName: mp.GetShortTagname(),
		Digest:    digest,
		Template:  "{{ .Prompt }}",
		License:   []string{},
168
169
170
	}

	for _, layer := range manifest.Layers {
Patrick Devine's avatar
Patrick Devine committed
171
		filename, err := GetBlobsPath(layer.Digest)
172
173
174
175
		if err != nil {
			return nil, err
		}

176
177
178
		switch layer.MediaType {
		case "application/vnd.ollama.image.model":
			model.ModelPath = filename
Patrick Devine's avatar
Patrick Devine committed
179
			model.OriginalModel = layer.From
180
		case "application/vnd.ollama.image.embed":
181
182
183
			// Deprecated in versions  > 0.1.2
			// TODO: remove this warning in a future version
			log.Print("WARNING: model contains embeddings, but embeddings in modelfiles have been deprecated and will be ignored.")
184
185
		case "application/vnd.ollama.image.adapter":
			model.AdapterPaths = append(model.AdapterPaths, filename)
186
187
188
189
190
191
192
193
194
		case "application/vnd.ollama.image.template":
			bts, err := os.ReadFile(filename)
			if err != nil {
				return nil, err
			}

			model.Template = string(bts)
		case "application/vnd.ollama.image.system":
			bts, err := os.ReadFile(filename)
195
196
197
			if err != nil {
				return nil, err
			}
198
199

			model.System = string(bts)
200
201
202
203
204
205
206
		case "application/vnd.ollama.image.prompt":
			bts, err := os.ReadFile(filename)
			if err != nil {
				return nil, err
			}

			model.Template = string(bts)
207
		case "application/vnd.ollama.image.params":
Michael Yang's avatar
Michael Yang committed
208
209
210
211
212
			params, err := os.Open(filename)
			if err != nil {
				return nil, err
			}
			defer params.Close()
213

214
			// parse model options parameters into a map so that we can see which fields have been specified explicitly
215
			if err = json.NewDecoder(params).Decode(&model.Options); err != nil {
216
217
				return nil, err
			}
Patrick Devine's avatar
Patrick Devine committed
218
219
220
221
222
223
		case "application/vnd.ollama.image.license":
			bts, err := os.ReadFile(filename)
			if err != nil {
				return nil, err
			}
			model.License = append(model.License, string(bts))
224
225
226
227
228
229
		}
	}

	return model, nil
}

230
231
func realpath(mfDir, from string) string {
	abspath, err := filepath.Abs(from)
Michael Yang's avatar
Michael Yang committed
232
	if err != nil {
233
		return from
234
235
	}

Michael Yang's avatar
Michael Yang committed
236
	home, err := os.UserHomeDir()
237
	if err != nil {
Michael Yang's avatar
Michael Yang committed
238
		return abspath
239
240
	}

241
	if from == "~" {
Michael Yang's avatar
Michael Yang committed
242
		return home
243
244
245
246
247
248
249
	} else if strings.HasPrefix(from, "~/") {
		return filepath.Join(home, from[2:])
	}

	if _, err := os.Stat(filepath.Join(mfDir, from)); err == nil {
		// this is a file relative to the Modelfile
		return filepath.Join(mfDir, from)
250
251
	}

Michael Yang's avatar
Michael Yang committed
252
253
254
	return abspath
}

255
func CreateModel(ctx context.Context, name, modelFileDir string, commands []parser.Command, fn func(resp api.ProgressResponse)) error {
256
257
	config := ConfigV2{
		OS:           "linux",
Michael Yang's avatar
Michael Yang committed
258
		Architecture: "amd64",
259
260
	}

Michael Yang's avatar
Michael Yang committed
261
262
	deleteMap := make(map[string]struct{})

Michael Yang's avatar
Michael Yang committed
263
	var layers []*LayerReader
Michael Yang's avatar
Michael Yang committed
264

265
	params := make(map[string][]string)
Michael Yang's avatar
Michael Yang committed
266
267
	fromParams := make(map[string]any)

268
	for _, c := range commands {
Michael Yang's avatar
Michael Yang committed
269
270
271
		log.Printf("[%s] - %s", c.Name, c.Args)
		mediatype := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name)

272
273
		switch c.Name {
		case "model":
Michael Yang's avatar
Michael Yang committed
274
275
276
277
278
279
280
281
282
			if strings.HasPrefix(c.Args, "@") {
				blobPath, err := GetBlobsPath(strings.TrimPrefix(c.Args, "@"))
				if err != nil {
					return err
				}

				c.Args = blobPath
			}

283
			bin, err := os.Open(realpath(modelFileDir, c.Args))
284
			if err != nil {
Michael Yang's avatar
Michael Yang committed
285
286
287
288
289
290
291
				// not a file on disk so must be a model reference
				modelpath := ParseModelPath(c.Args)
				manifest, _, err := GetManifest(modelpath)
				switch {
				case errors.Is(err, os.ErrNotExist):
					fn(api.ProgressResponse{Status: "pulling model"})
					if err := PullModel(ctx, c.Args, &RegistryOptions{}, fn); err != nil {
292
293
294
						return err
					}

Michael Yang's avatar
Michael Yang committed
295
					manifest, _, err = GetManifest(modelpath)
296
297
298
					if err != nil {
						return err
					}
Michael Yang's avatar
Michael Yang committed
299
300
				case err != nil:
					return err
301
				}
302

303
				fn(api.ProgressResponse{Status: "reading model metadata"})
Michael Yang's avatar
Michael Yang committed
304
				fromConfigPath, err := GetBlobsPath(manifest.Config.Digest)
Michael Yang's avatar
Michael Yang committed
305
306
307
308
				if err != nil {
					return err
				}

Michael Yang's avatar
Michael Yang committed
309
				fromConfigFile, err := os.Open(fromConfigPath)
Michael Yang's avatar
Michael Yang committed
310
311
312
				if err != nil {
					return err
				}
Michael Yang's avatar
Michael Yang committed
313
				defer fromConfigFile.Close()
Michael Yang's avatar
Michael Yang committed
314

Michael Yang's avatar
Michael Yang committed
315
316
				var fromConfig ConfigV2
				if err := json.NewDecoder(fromConfigFile).Decode(&fromConfig); err != nil {
Michael Yang's avatar
Michael Yang committed
317
318
319
					return err
				}

Michael Yang's avatar
Michael Yang committed
320
321
322
323
				config.ModelFormat = fromConfig.ModelFormat
				config.ModelFamily = fromConfig.ModelFamily
				config.ModelType = fromConfig.ModelType
				config.FileType = fromConfig.FileType
Michael Yang's avatar
Michael Yang committed
324

Michael Yang's avatar
Michael Yang committed
325
326
327
328
				for _, layer := range manifest.Layers {
					deleteMap[layer.Digest] = struct{}{}
					if layer.MediaType == "application/vnd.ollama.image.params" {
						fromParamsPath, err := GetBlobsPath(layer.Digest)
Michael Yang's avatar
Michael Yang committed
329
330
331
332
						if err != nil {
							return err
						}

Michael Yang's avatar
Michael Yang committed
333
						fromParamsFile, err := os.Open(fromParamsPath)
Michael Yang's avatar
Michael Yang committed
334
335
336
						if err != nil {
							return err
						}
Michael Yang's avatar
Michael Yang committed
337
						defer fromParamsFile.Close()
Michael Yang's avatar
Michael Yang committed
338

Michael Yang's avatar
Michael Yang committed
339
						if err := json.NewDecoder(fromParamsFile).Decode(&fromParams); err != nil {
Michael Yang's avatar
Michael Yang committed
340
341
342
343
							return err
						}
					}

Michael Yang's avatar
Michael Yang committed
344
					layer, err := GetLayerWithBufferFromLayer(layer)
345
346
347
					if err != nil {
						return err
					}
Michael Yang's avatar
Michael Yang committed
348
349
350

					layer.From = modelpath.GetShortTagname()
					layers = append(layers, layer)
351
				}
Michael Yang's avatar
Michael Yang committed
352
353
354

				deleteMap[manifest.Config.Digest] = struct{}{}
				continue
355
			}
Michael Yang's avatar
Michael Yang committed
356
			defer bin.Close()
357

Michael Yang's avatar
Michael Yang committed
358
359
			fn(api.ProgressResponse{Status: "creating model layer"})
			ggml, err := llm.DecodeGGML(bin)
Michael Yang's avatar
Michael Yang committed
360
361
			if err != nil {
				return err
362
363
			}

Michael Yang's avatar
Michael Yang committed
364
365
366
367
368
369
370
371
372
373
			config.ModelFormat = ggml.Name()
			config.ModelFamily = ggml.ModelFamily()
			config.ModelType = ggml.ModelType()
			config.FileType = ggml.FileType()

			bin.Seek(0, io.SeekStart)
			layer, err := CreateLayer(bin)
			if err != nil {
				return err
			}
374

Michael Yang's avatar
Michael Yang committed
375
376
377
378
			layer.MediaType = mediatype
			layers = append(layers, layer)
		case "adapter":
			fn(api.ProgressResponse{Status: "creating adapter layer"})
379
			bin, err := os.Open(realpath(modelFileDir, c.Args))
380
			if err != nil {
Michael Yang's avatar
Michael Yang committed
381
				return err
382
			}
Michael Yang's avatar
Michael Yang committed
383
			defer bin.Close()
384

Michael Yang's avatar
Michael Yang committed
385
			layer, err := CreateLayer(bin)
386
			if err != nil {
Michael Yang's avatar
Michael Yang committed
387
				return err
388
			}
Bruce MacDonald's avatar
Bruce MacDonald committed
389

Michael Yang's avatar
Michael Yang committed
390
391
392
393
394
395
			if layer.Size > 0 {
				layer.MediaType = mediatype
				layers = append(layers, layer)
			}
		case "license":
			fn(api.ProgressResponse{Status: "creating license layer"})
Bruce MacDonald's avatar
Bruce MacDonald committed
396
397
398
399
400
			layer, err := CreateLayer(strings.NewReader(c.Args))
			if err != nil {
				return err
			}

401
			if layer.Size > 0 {
Michael Yang's avatar
Michael Yang committed
402
				layer.MediaType = mediatype
403
404
				layers = append(layers, layer)
			}
Michael Yang's avatar
Michael Yang committed
405
406
407
		case "template", "system":
			fn(api.ProgressResponse{Status: fmt.Sprintf("creating %s layer", c.Name)})

Michael Yang's avatar
Michael Yang committed
408
			// remove duplicate layers
Michael Yang's avatar
Michael Yang committed
409
			layers = removeLayerFromLayers(layers, mediatype)
410

411
			layer, err := CreateLayer(strings.NewReader(c.Args))
412
			if err != nil {
413
				return err
414
			}
415

416
			if layer.Size > 0 {
Michael Yang's avatar
Michael Yang committed
417
				layer.MediaType = mediatype
418
419
				layers = append(layers, layer)
			}
420
		default:
421
			params[c.Name] = append(params[c.Name], c.Args)
422
423
424
		}
	}

Michael Yang's avatar
Michael Yang committed
425
	if len(params) > 0 {
Michael Yang's avatar
Michael Yang committed
426
		fn(api.ProgressResponse{Status: "creating parameters layer"})
Michael Yang's avatar
Michael Yang committed
427

428
		formattedParams, err := api.FormatParams(params)
429
		if err != nil {
Michael Yang's avatar
Michael Yang committed
430
			return err
431
		}
432

Michael Yang's avatar
Michael Yang committed
433
		for k, v := range fromParams {
Michael Yang's avatar
Michael Yang committed
434
435
436
437
438
			if _, ok := formattedParams[k]; !ok {
				formattedParams[k] = v
			}
		}

Michael Yang's avatar
Michael Yang committed
439
		if config.ModelType == "65B" {
Michael Yang's avatar
Michael Yang committed
440
			if gqa, ok := formattedParams["gqa"].(int); ok && gqa == 8 {
Michael Yang's avatar
Michael Yang committed
441
442
443
444
				config.ModelType = "70B"
			}
		}

Michael Yang's avatar
Michael Yang committed
445
446
		var b bytes.Buffer
		if err := json.NewEncoder(&b).Encode(formattedParams); err != nil {
447
448
449
			return err
		}

Michael Yang's avatar
Michael Yang committed
450
451
		fn(api.ProgressResponse{Status: "creating config layer"})
		layer, err := CreateLayer(bytes.NewReader(b.Bytes()))
452
		if err != nil {
Michael Yang's avatar
Michael Yang committed
453
			return err
454
		}
Michael Yang's avatar
Michael Yang committed
455
456
457

		layer.MediaType = "application/vnd.ollama.image.params"
		layers = append(layers, layer)
458
459
	}

460
461
462
463
464
	digests, err := getLayerDigests(layers)
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
465
	configLayer, err := createConfigLayer(config, digests)
466
467
468
	if err != nil {
		return err
	}
Michael Yang's avatar
Michael Yang committed
469
470
471

	layers = append(layers, configLayer)
	delete(deleteMap, configLayer.Digest)
472

Michael Yang's avatar
Michael Yang committed
473
	if err := SaveLayers(layers, fn, false); err != nil {
474
475
476
		return err
	}

Michael Yang's avatar
Michael Yang committed
477
478
479
480
481
482
	var contentLayers []*Layer
	for _, layer := range layers {
		contentLayers = append(contentLayers, &layer.Layer)
		delete(deleteMap, layer.Digest)
	}

483
	fn(api.ProgressResponse{Status: "writing manifest"})
Michael Yang's avatar
Michael Yang committed
484
	if err := CreateManifest(name, configLayer, contentLayers); err != nil {
485
486
487
		return err
	}

Michael Yang's avatar
Michael Yang committed
488
489
	if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		if err := deleteUnusedLayers(nil, deleteMap, false); err != nil {
490
491
492
493
			return err
		}
	}

494
	fn(api.ProgressResponse{Status: "success"})
495
496
497
	return nil
}

Michael Yang's avatar
Michael Yang committed
498
func removeLayerFromLayers(layers []*LayerReader, mediaType string) []*LayerReader {
Michael Yang's avatar
Michael Yang committed
499
500
501
	return slices.DeleteFunc(layers, func(layer *LayerReader) bool {
		return layer.MediaType == mediaType
	})
502
503
}

504
func SaveLayers(layers []*LayerReader, fn func(resp api.ProgressResponse), force bool) error {
505
506
	// Write each of the layers to disk
	for _, layer := range layers {
Patrick Devine's avatar
Patrick Devine committed
507
		fp, err := GetBlobsPath(layer.Digest)
508
509
510
		if err != nil {
			return err
		}
511
512

		_, err = os.Stat(fp)
513
		if os.IsNotExist(err) || force {
514
515
			fn(api.ProgressResponse{Status: fmt.Sprintf("writing layer %s", layer.Digest)})

516
517
518
519
520
521
522
			out, err := os.Create(fp)
			if err != nil {
				log.Printf("couldn't create %s", fp)
				return err
			}
			defer out.Close()

Michael Yang's avatar
Michael Yang committed
523
			if _, err = io.Copy(out, layer.Reader); err != nil {
524
525
				return err
			}
Michael Yang's avatar
Michael Yang committed
526

527
		} else {
528
			fn(api.ProgressResponse{Status: fmt.Sprintf("using already created layer %s", layer.Digest)})
529
530
531
532
533
534
		}
	}

	return nil
}

Michael Yang's avatar
Michael Yang committed
535
func CreateManifest(name string, cfg *LayerReader, layers []*Layer) error {
536
	mp := ParseModelPath(name)
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
	manifest := ManifestV2{
		SchemaVersion: 2,
		MediaType:     "application/vnd.docker.distribution.manifest.v2+json",
		Config: Layer{
			MediaType: cfg.MediaType,
			Size:      cfg.Size,
			Digest:    cfg.Digest,
		},
		Layers: layers,
	}

	manifestJSON, err := json.Marshal(manifest)
	if err != nil {
		return err
	}

553
	fp, err := mp.GetManifestPath()
554
555
556
	if err != nil {
		return err
	}
557
558
559
	if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
		return err
	}
560
	return os.WriteFile(fp, manifestJSON, 0o644)
561
562
}

Michael Yang's avatar
Michael Yang committed
563
func GetLayerWithBufferFromLayer(layer *Layer) (*LayerReader, error) {
Patrick Devine's avatar
Patrick Devine committed
564
	fp, err := GetBlobsPath(layer.Digest)
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
	if err != nil {
		return nil, err
	}

	file, err := os.Open(fp)
	if err != nil {
		return nil, fmt.Errorf("could not open blob: %w", err)
	}
	defer file.Close()

	newLayer, err := CreateLayer(file)
	if err != nil {
		return nil, err
	}
	newLayer.MediaType = layer.MediaType
	return newLayer, nil
}

Michael Yang's avatar
Michael Yang committed
583
func getLayerDigests(layers []*LayerReader) ([]string, error) {
584
585
586
587
588
589
590
591
592
593
594
	var digests []string
	for _, l := range layers {
		if l.Digest == "" {
			return nil, fmt.Errorf("layer is missing a digest")
		}
		digests = append(digests, l.Digest)
	}
	return digests, nil
}

// CreateLayer creates a Layer object from a given file
Michael Yang's avatar
Michael Yang committed
595
596
func CreateLayer(f io.ReadSeeker) (*LayerReader, error) {
	digest, size := GetSHA256Digest(f)
597
	f.Seek(0, io.SeekStart)
598

Michael Yang's avatar
Michael Yang committed
599
	layer := &LayerReader{
600
601
602
603
604
		Layer: Layer{
			MediaType: "application/vnd.docker.image.rootfs.diff.tar",
			Digest:    digest,
			Size:      size,
		},
Michael Yang's avatar
Michael Yang committed
605
		Reader: f,
606
607
608
609
610
	}

	return layer, nil
}

Patrick Devine's avatar
Patrick Devine committed
611
func CopyModel(src, dest string) error {
612
	srcModelPath := ParseModelPath(src)
613
	srcPath, err := srcModelPath.GetManifestPath()
614
615
616
617
	if err != nil {
		return err
	}

618
	destModelPath := ParseModelPath(dest)
619
	destPath, err := destModelPath.GetManifestPath()
Patrick Devine's avatar
Patrick Devine committed
620
621
622
	if err != nil {
		return err
	}
623
624
625
	if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
		return err
	}
Patrick Devine's avatar
Patrick Devine committed
626
627

	// copy the file
Michael Yang's avatar
Michael Yang committed
628
	input, err := os.ReadFile(srcPath)
Patrick Devine's avatar
Patrick Devine committed
629
630
631
632
633
	if err != nil {
		fmt.Println("Error reading file:", err)
		return err
	}

Michael Yang's avatar
Michael Yang committed
634
	err = os.WriteFile(destPath, input, 0o644)
Patrick Devine's avatar
Patrick Devine committed
635
636
637
638
639
640
641
642
	if err != nil {
		fmt.Println("Error reading file:", err)
		return err
	}

	return nil
}

Michael Yang's avatar
Michael Yang committed
643
func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]struct{}, dryRun bool) error {
644
645
646
647
	fp, err := GetManifestPath()
	if err != nil {
		return err
	}
Michael Yang's avatar
Michael Yang committed
648
649
650
651

	walkFunc := func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			return nil
652
653
		}

Michael Yang's avatar
Michael Yang committed
654
655
656
657
		dir, file := filepath.Split(path)
		dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
		tag := strings.Join([]string{dir, file}, ":")
		fmp := ParseModelPath(tag)
658

Michael Yang's avatar
Michael Yang committed
659
		// skip the manifest we're trying to delete
660
		if skipModelPath != nil && skipModelPath.GetFullTagname() == fmp.GetFullTagname() {
Michael Yang's avatar
Michael Yang committed
661
			return nil
662
		}
Michael Yang's avatar
Michael Yang committed
663
664
665
666
667
668
669
670
671
672
673
674

		// save (i.e. delete from the deleteMap) any files used in other manifests
		manifest, _, err := GetManifest(fmp)
		if err != nil {
			return nil
		}

		for _, layer := range manifest.Layers {
			delete(deleteMap, layer.Digest)
		}

		delete(deleteMap, manifest.Config.Digest)
675
		return nil
Michael Yang's avatar
Michael Yang committed
676
677
678
	}

	if err := filepath.Walk(fp, walkFunc); err != nil {
Michael Yang's avatar
Michael Yang committed
679
680
		return err
	}
681
682

	// only delete the files which are still in the deleteMap
Michael Yang's avatar
Michael Yang committed
683
684
685
686
687
688
689
690
691
	for k := range deleteMap {
		fp, err := GetBlobsPath(k)
		if err != nil {
			log.Printf("couldn't get file path for '%s': %v", k, err)
			continue
		}
		if !dryRun {
			if err := os.Remove(fp); err != nil {
				log.Printf("couldn't remove file '%s': %v", fp, err)
692
693
				continue
			}
Michael Yang's avatar
Michael Yang committed
694
695
		} else {
			log.Printf("wanted to remove: %s", fp)
696
697
698
		}
	}

699
700
701
702
	return nil
}

func PruneLayers() error {
Michael Yang's avatar
Michael Yang committed
703
	deleteMap := make(map[string]struct{})
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
	p, err := GetBlobsPath("")
	if err != nil {
		return err
	}

	blobs, err := os.ReadDir(p)
	if err != nil {
		log.Printf("couldn't read dir '%s': %v", p, err)
		return err
	}

	for _, blob := range blobs {
		name := blob.Name()
		if runtime.GOOS == "windows" {
			name = strings.ReplaceAll(name, "-", ":")
		}
Michael Yang's avatar
Michael Yang committed
720
721
722
		if strings.HasPrefix(name, "sha256:") {
			deleteMap[name] = struct{}{}
		}
723
724
725
726
727
728
729
730
731
732
733
734
735
736
	}

	log.Printf("total blobs: %d", len(deleteMap))

	err = deleteUnusedLayers(nil, deleteMap, false)
	if err != nil {
		return err
	}

	log.Printf("total unused blobs removed: %d", len(deleteMap))

	return nil
}

Michael Yang's avatar
Michael Yang committed
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
func PruneDirectory(path string) error {
	info, err := os.Lstat(path)
	if err != nil {
		return err
	}

	if info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
		entries, err := os.ReadDir(path)
		if err != nil {
			return err
		}

		for _, entry := range entries {
			if err := PruneDirectory(filepath.Join(path, entry.Name())); err != nil {
				return err
			}
		}

		entries, err = os.ReadDir(path)
		if err != nil {
			return err
		}

		if len(entries) > 0 {
			return nil
		}

		return os.Remove(path)
	}

	return nil
}

770
771
772
773
774
775
776
func DeleteModel(name string) error {
	mp := ParseModelPath(name)
	manifest, _, err := GetManifest(mp)
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
777
	deleteMap := make(map[string]struct{})
778
	for _, layer := range manifest.Layers {
Michael Yang's avatar
Michael Yang committed
779
		deleteMap[layer.Digest] = struct{}{}
780
	}
Michael Yang's avatar
Michael Yang committed
781
	deleteMap[manifest.Config.Digest] = struct{}{}
782
783
784
785
786
787

	err = deleteUnusedLayers(&mp, deleteMap, false)
	if err != nil {
		return err
	}

788
	fp, err := mp.GetManifestPath()
789
790
791
792
793
794
795
796
797
798
799
800
	if err != nil {
		return err
	}
	err = os.Remove(fp)
	if err != nil {
		log.Printf("couldn't remove manifest file '%s': %v", fp, err)
		return err
	}

	return nil
}

Patrick Devine's avatar
Patrick Devine committed
801
func ShowModelfile(model *Model) (string, error) {
Michael Yang's avatar
Michael Yang committed
802
	var mt struct {
Patrick Devine's avatar
Patrick Devine committed
803
		*Model
Michael Yang's avatar
Michael Yang committed
804
		From       string
Michael Yang's avatar
Michael Yang committed
805
		Parameters map[string][]any
Patrick Devine's avatar
Patrick Devine committed
806
807
	}

Michael Yang's avatar
Michael Yang committed
808
	mt.Parameters = make(map[string][]any)
Patrick Devine's avatar
Patrick Devine committed
809
	for k, v := range model.Options {
Michael Yang's avatar
Michael Yang committed
810
811
812
		if s, ok := v.([]any); ok {
			mt.Parameters[k] = s
			continue
Patrick Devine's avatar
Patrick Devine committed
813
814
		}

Michael Yang's avatar
Michael Yang committed
815
		mt.Parameters[k] = []any{v}
Patrick Devine's avatar
Patrick Devine committed
816
817
	}

Michael Yang's avatar
Michael Yang committed
818
819
	mt.Model = model
	mt.From = model.ModelPath
Patrick Devine's avatar
Patrick Devine committed
820

Michael Yang's avatar
Michael Yang committed
821
	if model.OriginalModel != "" {
Daniel Reis's avatar
Daniel Reis committed
822
		mt.From = model.OriginalModel
Patrick Devine's avatar
Patrick Devine committed
823
824
825
826
827
828
829
830
	}

	modelFile := `# Modelfile generated by "ollama show"
# To build a new Modelfile based on this one, replace the FROM line with:
# FROM {{ .ShortName }}

FROM {{ .From }}
TEMPLATE """{{ .Template }}"""
831
832

{{- if .System }}
Patrick Devine's avatar
Patrick Devine committed
833
SYSTEM """{{ .System }}"""
834
{{- end }}
835
836
837
838

{{- range $adapter := .AdapterPaths }}
ADAPTER {{ $adapter }}
{{- end }}
Michael Yang's avatar
Michael Yang committed
839

Michael Yang's avatar
Michael Yang committed
840
841
842
843
{{- range $k, $v := .Parameters }}
{{- range $parameter := $v }}
PARAMETER {{ $k }} {{ printf "%#v" $parameter }}
{{- end }}
Michael Yang's avatar
Michael Yang committed
844
{{- end }}`
Patrick Devine's avatar
Patrick Devine committed
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861

	tmpl, err := template.New("").Parse(modelFile)
	if err != nil {
		log.Printf("error parsing template: %q", err)
		return "", err
	}

	var buf bytes.Buffer

	if err = tmpl.Execute(&buf, mt); err != nil {
		log.Printf("error executing template: %q", err)
		return "", err
	}

	return buf.String(), nil
}

862
func PushModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
863
	mp := ParseModelPath(name)
864
865
	fn(api.ProgressResponse{Status: "retrieving manifest"})

866
867
868
869
	if mp.ProtocolScheme == "http" && !regOpts.Insecure {
		return fmt.Errorf("insecure protocol http")
	}

Patrick Devine's avatar
Patrick Devine committed
870
	manifest, _, err := GetManifest(mp)
871
	if err != nil {
872
		fn(api.ProgressResponse{Status: "couldn't retrieve manifest"})
873
874
875
876
		return err
	}

	var layers []*Layer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
877
	layers = append(layers, manifest.Layers...)
878
879
880
	layers = append(layers, &manifest.Config)

	for _, layer := range layers {
Michael Yang's avatar
Michael Yang committed
881
		if err := uploadBlob(ctx, mp, layer, regOpts, fn); err != nil {
882
			log.Printf("error uploading blob: %v", err)
883
884
885
			if errors.Is(err, errUnauthorized) {
				return fmt.Errorf("unable to push %s, make sure this namespace exists and you are authorized to push to it", ParseModelPath(name).GetNamespaceRepository())
			}
886
887
			return err
		}
888
889
	}

890
	fn(api.ProgressResponse{Status: "pushing manifest"})
Michael Yang's avatar
Michael Yang committed
891
892
	requestURL := mp.BaseURL()
	requestURL = requestURL.JoinPath("v2", mp.GetNamespaceRepository(), "manifests", mp.Tag)
893
894
895
896
897
898

	manifestJSON, err := json.Marshal(manifest)
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
899
900
	headers := make(http.Header)
	headers.Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
Michael Yang's avatar
Michael Yang committed
901
	resp, err := makeRequestWithRetry(ctx, http.MethodPut, requestURL, headers, bytes.NewReader(manifestJSON), regOpts)
902
903
904
905
906
	if err != nil {
		return err
	}
	defer resp.Body.Close()

907
	fn(api.ProgressResponse{Status: "success"})
908
909
910
911

	return nil
}

912
func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
913
914
	mp := ParseModelPath(name)

915
916
917
918
919
	var manifest *ManifestV2
	var err error
	var noprune string

	// build deleteMap to prune unused layers
Michael Yang's avatar
Michael Yang committed
920
	deleteMap := make(map[string]struct{})
921
922
923
924
925
926
927
928
929

	if noprune = os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		manifest, _, err = GetManifest(mp)
		if err != nil && !errors.Is(err, os.ErrNotExist) {
			return err
		}

		if manifest != nil {
			for _, l := range manifest.Layers {
Michael Yang's avatar
Michael Yang committed
930
				deleteMap[l.Digest] = struct{}{}
931
			}
Michael Yang's avatar
Michael Yang committed
932
			deleteMap[manifest.Config.Digest] = struct{}{}
933
934
935
		}
	}

936
937
	if mp.ProtocolScheme == "http" && !regOpts.Insecure {
		return fmt.Errorf("insecure protocol http")
938
	}
939

940
	fn(api.ProgressResponse{Status: "pulling manifest"})
941

942
	manifest, err = pullModelManifest(ctx, mp, regOpts)
943
	if err != nil {
944
		return fmt.Errorf("pull model manifest: %s", err)
945
946
947
	}

	var layers []*Layer
Bruce MacDonald's avatar
Bruce MacDonald committed
948
	layers = append(layers, manifest.Layers...)
949
950
951
	layers = append(layers, &manifest.Config)

	for _, layer := range layers {
952
953
954
955
956
957
958
959
		if err := downloadBlob(
			ctx,
			downloadOpts{
				mp:      mp,
				digest:  layer.Digest,
				regOpts: regOpts,
				fn:      fn,
			}); err != nil {
960
961
			return err
		}
962
		delete(deleteMap, layer.Digest)
963
	}
964
	delete(deleteMap, manifest.Config.Digest)
965

Michael Yang's avatar
Michael Yang committed
966
967
968
	fn(api.ProgressResponse{Status: "verifying sha256 digest"})
	for _, layer := range layers {
		if err := verifyBlob(layer.Digest); err != nil {
969
970
971
972
973
974
975
976
977
978
979
			if errors.Is(err, errDigestMismatch) {
				// something went wrong, delete the blob
				fp, err := GetBlobsPath(layer.Digest)
				if err != nil {
					return err
				}
				if err := os.Remove(fp); err != nil {
					// log this, but return the original error
					log.Printf("couldn't remove file with digest mismatch '%s': %v", fp, err)
				}
			}
Michael Yang's avatar
Michael Yang committed
980
981
982
983
			return err
		}
	}

984
	fn(api.ProgressResponse{Status: "writing manifest"})
985

986
	manifestJSON, err := json.Marshal(manifest)
987
988
989
990
	if err != nil {
		return err
	}

991
	fp, err := mp.GetManifestPath()
992
993
994
	if err != nil {
		return err
	}
995
996
997
	if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
		return err
	}
998

Bruce MacDonald's avatar
Bruce MacDonald committed
999
	err = os.WriteFile(fp, manifestJSON, 0o644)
1000
1001
1002
1003
1004
	if err != nil {
		log.Printf("couldn't write to %s", fp)
		return err
	}

1005
1006
1007
1008
1009
1010
1011
1012
	if noprune == "" {
		fn(api.ProgressResponse{Status: "removing any unused layers"})
		err = deleteUnusedLayers(nil, deleteMap, false)
		if err != nil {
			return err
		}
	}

1013
	fn(api.ProgressResponse{Status: "success"})
1014
1015
1016
1017

	return nil
}

1018
func pullModelManifest(ctx context.Context, mp ModelPath, regOpts *RegistryOptions) (*ManifestV2, error) {
Michael Yang's avatar
Michael Yang committed
1019
	requestURL := mp.BaseURL().JoinPath("v2", mp.GetNamespaceRepository(), "manifests", mp.Tag)
1020

Michael Yang's avatar
Michael Yang committed
1021
1022
	headers := make(http.Header)
	headers.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json")
Michael Yang's avatar
Michael Yang committed
1023
	resp, err := makeRequestWithRetry(ctx, http.MethodGet, requestURL, headers, nil, regOpts)
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var m *ManifestV2
	if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
		return nil, err
	}

	return m, err
}

1037
1038
1039
1040
func createConfigLayer(config ConfigV2, layers []string) (*LayerReader, error) {
	config.RootFS = RootFS{
		Type:    "layers",
		DiffIDs: layers,
1041
1042
1043
1044
1045
1046
1047
	}

	configJSON, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

1048
	digest, size := GetSHA256Digest(bytes.NewBuffer(configJSON))
1049

Michael Yang's avatar
Michael Yang committed
1050
	layer := &LayerReader{
1051
1052
1053
1054
1055
		Layer: Layer{
			MediaType: "application/vnd.docker.container.image.v1+json",
			Digest:    digest,
			Size:      size,
		},
1056
		Reader: bytes.NewBuffer(configJSON),
1057
1058
1059
1060
1061
	}
	return layer, nil
}

// GetSHA256Digest returns the SHA256 hash of a given buffer and returns it, and the size of buffer
Michael Yang's avatar
Michael Yang committed
1062
func GetSHA256Digest(r io.Reader) (string, int64) {
Michael Yang's avatar
Michael Yang committed
1063
1064
1065
1066
1067
1068
	h := sha256.New()
	n, err := io.Copy(h, r)
	if err != nil {
		log.Fatal(err)
	}

Michael Yang's avatar
Michael Yang committed
1069
	return fmt.Sprintf("sha256:%x", h.Sum(nil)), n
1070
1071
}

1072
1073
var errUnauthorized = fmt.Errorf("unauthorized")

Michael Yang's avatar
Michael Yang committed
1074
func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.ReadSeeker, regOpts *RegistryOptions) (*http.Response, error) {
1075
1076
1077
1078
1079
	resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts)
	if err != nil {
		if !errors.Is(err, context.Canceled) {
			log.Printf("request failed: %v", err)
		}
1080

1081
1082
1083
1084
1085
1086
1087
1088
1089
		return nil, err
	}

	switch {
	case resp.StatusCode == http.StatusUnauthorized:
		// Handle authentication error with one retry
		auth := resp.Header.Get("www-authenticate")
		authRedir := ParseAuthRedirectString(auth)
		token, err := getAuthToken(ctx, authRedir)
Michael Yang's avatar
Michael Yang committed
1090
1091
1092
		if err != nil {
			return nil, err
		}
1093
1094
1095
		regOpts.Token = token
		if body != nil {
			_, err = body.Seek(0, io.SeekStart)
Michael Yang's avatar
Michael Yang committed
1096
1097
1098
1099
			if err != nil {
				return nil, err
			}
		}
1100
1101
1102
1103
1104
1105
1106

		resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts)
		if resp.StatusCode == http.StatusUnauthorized {
			return nil, errUnauthorized
		}

		return resp, err
1107
1108
1109
1110
1111
1112
1113
1114
	case resp.StatusCode == http.StatusNotFound:
		return nil, os.ErrNotExist
	case resp.StatusCode >= http.StatusBadRequest:
		responseBody, err := io.ReadAll(resp.Body)
		if err != nil {
			return nil, fmt.Errorf("%d: %s", resp.StatusCode, err)
		}
		return nil, fmt.Errorf("%d: %s", resp.StatusCode, responseBody)
Michael Yang's avatar
Michael Yang committed
1115
1116
	}

1117
	return resp, nil
Michael Yang's avatar
Michael Yang committed
1118
1119
}

Michael Yang's avatar
Michael Yang committed
1120
func makeRequest(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.Reader, regOpts *RegistryOptions) (*http.Response, error) {
Michael Yang's avatar
Michael Yang committed
1121
	if requestURL.Scheme != "http" && regOpts != nil && regOpts.Insecure {
Michael Yang's avatar
Michael Yang committed
1122
		requestURL.Scheme = "http"
1123
1124
	}

Michael Yang's avatar
Michael Yang committed
1125
	req, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
1126
1127
1128
1129
	if err != nil {
		return nil, err
	}

Michael Yang's avatar
Michael Yang committed
1130
1131
1132
1133
	if headers != nil {
		req.Header = headers
	}

Michael Yang's avatar
Michael Yang committed
1134
1135
1136
1137
1138
1139
	if regOpts != nil {
		if regOpts.Token != "" {
			req.Header.Set("Authorization", "Bearer "+regOpts.Token)
		} else if regOpts.Username != "" && regOpts.Password != "" {
			req.SetBasicAuth(regOpts.Username, regOpts.Password)
		}
1140
1141
	}

Michael Yang's avatar
Michael Yang committed
1142
	req.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
1143

Michael Yang's avatar
Michael Yang committed
1144
1145
1146
1147
1148
1149
1150
1151
1152
	if s := req.Header.Get("Content-Length"); s != "" {
		contentLength, err := strconv.ParseInt(s, 10, 64)
		if err != nil {
			return nil, err
		}

		req.ContentLength = contentLength
	}

Michael Yang's avatar
Michael Yang committed
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
	proxyURL, err := http.ProxyFromEnvironment(req)
	if err != nil {
		return nil, err
	}

	client := http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyURL(proxyURL),
		},
	}

	resp, err := client.Do(req)
1165
1166
1167
1168
1169
1170
	if err != nil {
		return nil, err
	}

	return resp, nil
}
Michael Yang's avatar
Michael Yang committed
1171

Patrick Devine's avatar
Patrick Devine committed
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
func getValue(header, key string) string {
	startIdx := strings.Index(header, key+"=")
	if startIdx == -1 {
		return ""
	}

	// Move the index to the starting quote after the key.
	startIdx += len(key) + 2
	endIdx := startIdx

	for endIdx < len(header) {
		if header[endIdx] == '"' {
			if endIdx+1 < len(header) && header[endIdx+1] != ',' { // If the next character isn't a comma, continue
				endIdx++
				continue
			}
			break
		}
		endIdx++
	}
	return header[startIdx:endIdx]
}

func ParseAuthRedirectString(authStr string) AuthRedirect {
	authStr = strings.TrimPrefix(authStr, "Bearer ")

	return AuthRedirect{
		Realm:   getValue(authStr, "realm"),
		Service: getValue(authStr, "service"),
		Scope:   getValue(authStr, "scope"),
	}
}

1205
1206
var errDigestMismatch = fmt.Errorf("digest mismatch, file must be downloaded again")

Michael Yang's avatar
Michael Yang committed
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
func verifyBlob(digest string) error {
	fp, err := GetBlobsPath(digest)
	if err != nil {
		return err
	}

	f, err := os.Open(fp)
	if err != nil {
		return err
	}
	defer f.Close()

	fileDigest, _ := GetSHA256Digest(f)
	if digest != fileDigest {
1221
		return fmt.Errorf("%w: want %s, got %s", errDigestMismatch, digest, fileDigest)
Michael Yang's avatar
Michael Yang committed
1222
1223
1224
1225
	}

	return nil
}