images.go 28.6 KB
Newer Older
1
2
3
4
package server

import (
	"bytes"
5
	"context"
6
	"crypto/sha256"
Patrick Devine's avatar
Patrick Devine committed
7
	"encoding/hex"
8
9
10
11
12
13
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
Michael Yang's avatar
Michael Yang committed
14
	"net/url"
15
16
	"os"
	"path/filepath"
Michael Yang's avatar
Michael Yang committed
17
	"reflect"
Michael Yang's avatar
Michael Yang committed
18
	"runtime"
19
20
	"strconv"
	"strings"
Quinn Slack's avatar
Quinn Slack committed
21
	"text/template"
22

Michael Yang's avatar
Michael Yang committed
23
24
	"golang.org/x/exp/slices"

25
	"github.com/jmorganca/ollama/api"
26
	"github.com/jmorganca/ollama/llm"
27
	"github.com/jmorganca/ollama/parser"
Michael Yang's avatar
Michael Yang committed
28
	"github.com/jmorganca/ollama/version"
29
30
)

31
32
33
34
type RegistryOptions struct {
	Insecure bool
	Username string
	Password string
Patrick Devine's avatar
Patrick Devine committed
35
	Token    string
36
37
}

38
type Model struct {
Patrick Devine's avatar
Patrick Devine committed
39
40
41
42
43
44
45
46
47
48
	Name          string `json:"name"`
	ShortName     string
	ModelPath     string
	OriginalModel string
	AdapterPaths  []string
	Template      string
	System        string
	License       []string
	Digest        string
	Options       map[string]interface{}
49
50
}

51
func (m *Model) Prompt(request api.GenerateRequest) (string, error) {
52
53
54
55
56
57
	t := m.Template
	if request.Template != "" {
		t = request.Template
	}

	tmpl, err := template.New("").Parse(t)
58
59
60
61
62
	if err != nil {
		return "", err
	}

	var vars struct {
Michael Yang's avatar
Michael Yang committed
63
		First  bool
64
65
66
67
		System string
		Prompt string
	}

Michael Yang's avatar
Michael Yang committed
68
	vars.First = len(request.Context) == 0
69
	vars.System = m.System
70
71
	vars.Prompt = request.Prompt

72
73
74
75
	if request.System != "" {
		vars.System = request.System
	}

76
77
78
79
80
81
82
83
	var sb strings.Builder
	if err := tmpl.Execute(&sb, vars); err != nil {
		return "", err
	}

	return sb.String(), nil
}

84
85
86
87
88
89
90
91
92
93
type ManifestV2 struct {
	SchemaVersion int      `json:"schemaVersion"`
	MediaType     string   `json:"mediaType"`
	Config        Layer    `json:"config"`
	Layers        []*Layer `json:"layers"`
}

type Layer struct {
	MediaType string `json:"mediaType"`
	Digest    string `json:"digest"`
Michael Yang's avatar
Michael Yang committed
94
	Size      int64  `json:"size"`
Michael Yang's avatar
Michael Yang committed
95
	From      string `json:"from,omitempty"`
96
97
}

Michael Yang's avatar
Michael Yang committed
98
type LayerReader struct {
99
	Layer
Michael Yang's avatar
Michael Yang committed
100
	io.Reader
101
102
103
}

type ConfigV2 struct {
Michael Yang's avatar
Michael Yang committed
104
105
106
107
108
	ModelFormat string `json:"model_format"`
	ModelFamily string `json:"model_family"`
	ModelType   string `json:"model_type"`
	FileType    string `json:"file_type"`
	RootFS      RootFS `json:"rootfs"`
109
110

	// required by spec
111
112
113
114
115
116
117
118
119
	Architecture string `json:"architecture"`
	OS           string `json:"os"`
}

type RootFS struct {
	Type    string   `json:"type"`
	DiffIDs []string `json:"diff_ids"`
}

Michael Yang's avatar
Michael Yang committed
120
func (m *ManifestV2) GetTotalSize() (total int64) {
Patrick Devine's avatar
Patrick Devine committed
121
122
123
	for _, layer := range m.Layers {
		total += layer.Size
	}
Michael Yang's avatar
Michael Yang committed
124

Patrick Devine's avatar
Patrick Devine committed
125
126
127
128
	total += m.Config.Size
	return total
}

Patrick Devine's avatar
Patrick Devine committed
129
func GetManifest(mp ModelPath) (*ManifestV2, string, error) {
130
	fp, err := mp.GetManifestPath()
131
	if err != nil {
Patrick Devine's avatar
Patrick Devine committed
132
		return nil, "", err
133
	}
134

135
	if _, err = os.Stat(fp); err != nil {
Patrick Devine's avatar
Patrick Devine committed
136
		return nil, "", err
137
138
139
140
	}

	var manifest *ManifestV2

141
	bts, err := os.ReadFile(fp)
142
	if err != nil {
Patrick Devine's avatar
Patrick Devine committed
143
		return nil, "", fmt.Errorf("couldn't open file '%s'", fp)
144
145
	}

Patrick Devine's avatar
Patrick Devine committed
146
147
148
	shaSum := sha256.Sum256(bts)
	shaStr := hex.EncodeToString(shaSum[:])

149
	if err := json.Unmarshal(bts, &manifest); err != nil {
Patrick Devine's avatar
Patrick Devine committed
150
		return nil, "", err
151
152
	}

Patrick Devine's avatar
Patrick Devine committed
153
	return manifest, shaStr, nil
154
155
156
}

func GetModel(name string) (*Model, error) {
157
	mp := ParseModelPath(name)
Patrick Devine's avatar
Patrick Devine committed
158
	manifest, digest, err := GetManifest(mp)
159
160
161
162
163
	if err != nil {
		return nil, err
	}

	model := &Model{
164
165
166
167
168
		Name:      mp.GetFullTagname(),
		ShortName: mp.GetShortTagname(),
		Digest:    digest,
		Template:  "{{ .Prompt }}",
		License:   []string{},
169
170
171
	}

	for _, layer := range manifest.Layers {
Patrick Devine's avatar
Patrick Devine committed
172
		filename, err := GetBlobsPath(layer.Digest)
173
174
175
176
		if err != nil {
			return nil, err
		}

177
178
179
		switch layer.MediaType {
		case "application/vnd.ollama.image.model":
			model.ModelPath = filename
Patrick Devine's avatar
Patrick Devine committed
180
			model.OriginalModel = layer.From
181
		case "application/vnd.ollama.image.embed":
182
183
184
			// Deprecated in versions  > 0.1.2
			// TODO: remove this warning in a future version
			log.Print("WARNING: model contains embeddings, but embeddings in modelfiles have been deprecated and will be ignored.")
185
186
		case "application/vnd.ollama.image.adapter":
			model.AdapterPaths = append(model.AdapterPaths, filename)
187
188
189
190
191
192
193
194
195
		case "application/vnd.ollama.image.template":
			bts, err := os.ReadFile(filename)
			if err != nil {
				return nil, err
			}

			model.Template = string(bts)
		case "application/vnd.ollama.image.system":
			bts, err := os.ReadFile(filename)
196
197
198
			if err != nil {
				return nil, err
			}
199
200

			model.System = string(bts)
201
202
203
204
205
206
207
		case "application/vnd.ollama.image.prompt":
			bts, err := os.ReadFile(filename)
			if err != nil {
				return nil, err
			}

			model.Template = string(bts)
208
		case "application/vnd.ollama.image.params":
Michael Yang's avatar
Michael Yang committed
209
210
211
212
213
			params, err := os.Open(filename)
			if err != nil {
				return nil, err
			}
			defer params.Close()
214

215
			// parse model options parameters into a map so that we can see which fields have been specified explicitly
216
			if err = json.NewDecoder(params).Decode(&model.Options); err != nil {
217
218
				return nil, err
			}
Patrick Devine's avatar
Patrick Devine committed
219
220
221
222
223
224
		case "application/vnd.ollama.image.license":
			bts, err := os.ReadFile(filename)
			if err != nil {
				return nil, err
			}
			model.License = append(model.License, string(bts))
225
226
227
228
229
230
		}
	}

	return model, nil
}

Michael Yang's avatar
Michael Yang committed
231
232
233
234
func realpath(p string) string {
	abspath, err := filepath.Abs(p)
	if err != nil {
		return p
235
236
	}

Michael Yang's avatar
Michael Yang committed
237
	home, err := os.UserHomeDir()
238
	if err != nil {
Michael Yang's avatar
Michael Yang committed
239
		return abspath
240
241
	}

Michael Yang's avatar
Michael Yang committed
242
243
244
245
	if p == "~" {
		return home
	} else if strings.HasPrefix(p, "~/") {
		return filepath.Join(home, p[2:])
246
247
	}

Michael Yang's avatar
Michael Yang committed
248
249
250
251
	return abspath
}

func CreateModel(ctx context.Context, name string, commands []parser.Command, fn func(resp api.ProgressResponse)) error {
252
253
	config := ConfigV2{
		OS:           "linux",
Michael Yang's avatar
Michael Yang committed
254
		Architecture: "amd64",
255
256
	}

Michael Yang's avatar
Michael Yang committed
257
258
	deleteMap := make(map[string]struct{})

Michael Yang's avatar
Michael Yang committed
259
	var layers []*LayerReader
Michael Yang's avatar
Michael Yang committed
260

261
	params := make(map[string][]string)
Michael Yang's avatar
Michael Yang committed
262
263
	fromParams := make(map[string]any)

264
	for _, c := range commands {
Michael Yang's avatar
Michael Yang committed
265
266
267
		log.Printf("[%s] - %s", c.Name, c.Args)
		mediatype := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name)

268
269
		switch c.Name {
		case "model":
Michael Yang's avatar
Michael Yang committed
270
271
272
273
274
275
276
277
278
			if strings.HasPrefix(c.Args, "@") {
				blobPath, err := GetBlobsPath(strings.TrimPrefix(c.Args, "@"))
				if err != nil {
					return err
				}

				c.Args = blobPath
			}

Michael Yang's avatar
Michael Yang committed
279
			bin, err := os.Open(realpath(c.Args))
280
			if err != nil {
Michael Yang's avatar
Michael Yang committed
281
282
283
284
285
286
287
				// not a file on disk so must be a model reference
				modelpath := ParseModelPath(c.Args)
				manifest, _, err := GetManifest(modelpath)
				switch {
				case errors.Is(err, os.ErrNotExist):
					fn(api.ProgressResponse{Status: "pulling model"})
					if err := PullModel(ctx, c.Args, &RegistryOptions{}, fn); err != nil {
288
289
290
						return err
					}

Michael Yang's avatar
Michael Yang committed
291
					manifest, _, err = GetManifest(modelpath)
292
293
294
					if err != nil {
						return err
					}
Michael Yang's avatar
Michael Yang committed
295
296
				case err != nil:
					return err
297
				}
298

299
				fn(api.ProgressResponse{Status: "reading model metadata"})
Michael Yang's avatar
Michael Yang committed
300
				fromConfigPath, err := GetBlobsPath(manifest.Config.Digest)
Michael Yang's avatar
Michael Yang committed
301
302
303
304
				if err != nil {
					return err
				}

Michael Yang's avatar
Michael Yang committed
305
				fromConfigFile, err := os.Open(fromConfigPath)
Michael Yang's avatar
Michael Yang committed
306
307
308
				if err != nil {
					return err
				}
Michael Yang's avatar
Michael Yang committed
309
				defer fromConfigFile.Close()
Michael Yang's avatar
Michael Yang committed
310

Michael Yang's avatar
Michael Yang committed
311
312
				var fromConfig ConfigV2
				if err := json.NewDecoder(fromConfigFile).Decode(&fromConfig); err != nil {
Michael Yang's avatar
Michael Yang committed
313
314
315
					return err
				}

Michael Yang's avatar
Michael Yang committed
316
317
318
319
				config.ModelFormat = fromConfig.ModelFormat
				config.ModelFamily = fromConfig.ModelFamily
				config.ModelType = fromConfig.ModelType
				config.FileType = fromConfig.FileType
Michael Yang's avatar
Michael Yang committed
320

Michael Yang's avatar
Michael Yang committed
321
322
323
324
				for _, layer := range manifest.Layers {
					deleteMap[layer.Digest] = struct{}{}
					if layer.MediaType == "application/vnd.ollama.image.params" {
						fromParamsPath, err := GetBlobsPath(layer.Digest)
Michael Yang's avatar
Michael Yang committed
325
326
327
328
						if err != nil {
							return err
						}

Michael Yang's avatar
Michael Yang committed
329
						fromParamsFile, err := os.Open(fromParamsPath)
Michael Yang's avatar
Michael Yang committed
330
331
332
						if err != nil {
							return err
						}
Michael Yang's avatar
Michael Yang committed
333
						defer fromParamsFile.Close()
Michael Yang's avatar
Michael Yang committed
334

Michael Yang's avatar
Michael Yang committed
335
						if err := json.NewDecoder(fromParamsFile).Decode(&fromParams); err != nil {
Michael Yang's avatar
Michael Yang committed
336
337
338
339
							return err
						}
					}

Michael Yang's avatar
Michael Yang committed
340
					layer, err := GetLayerWithBufferFromLayer(layer)
341
342
343
					if err != nil {
						return err
					}
Michael Yang's avatar
Michael Yang committed
344
345
346

					layer.From = modelpath.GetShortTagname()
					layers = append(layers, layer)
347
				}
Michael Yang's avatar
Michael Yang committed
348
349
350

				deleteMap[manifest.Config.Digest] = struct{}{}
				continue
351
			}
Michael Yang's avatar
Michael Yang committed
352
			defer bin.Close()
353

Michael Yang's avatar
Michael Yang committed
354
355
			fn(api.ProgressResponse{Status: "creating model layer"})
			ggml, err := llm.DecodeGGML(bin)
Michael Yang's avatar
Michael Yang committed
356
357
			if err != nil {
				return err
358
359
			}

Michael Yang's avatar
Michael Yang committed
360
361
362
363
364
365
366
367
368
369
			config.ModelFormat = ggml.Name()
			config.ModelFamily = ggml.ModelFamily()
			config.ModelType = ggml.ModelType()
			config.FileType = ggml.FileType()

			bin.Seek(0, io.SeekStart)
			layer, err := CreateLayer(bin)
			if err != nil {
				return err
			}
370

Michael Yang's avatar
Michael Yang committed
371
372
373
374
375
			layer.MediaType = mediatype
			layers = append(layers, layer)
		case "adapter":
			fn(api.ProgressResponse{Status: "creating adapter layer"})
			bin, err := os.Open(realpath(c.Args))
376
			if err != nil {
Michael Yang's avatar
Michael Yang committed
377
				return err
378
			}
Michael Yang's avatar
Michael Yang committed
379
			defer bin.Close()
380

Michael Yang's avatar
Michael Yang committed
381
			layer, err := CreateLayer(bin)
382
			if err != nil {
Michael Yang's avatar
Michael Yang committed
383
				return err
384
			}
Bruce MacDonald's avatar
Bruce MacDonald committed
385

Michael Yang's avatar
Michael Yang committed
386
387
388
389
390
391
			if layer.Size > 0 {
				layer.MediaType = mediatype
				layers = append(layers, layer)
			}
		case "license":
			fn(api.ProgressResponse{Status: "creating license layer"})
Bruce MacDonald's avatar
Bruce MacDonald committed
392
393
394
395
396
			layer, err := CreateLayer(strings.NewReader(c.Args))
			if err != nil {
				return err
			}

397
			if layer.Size > 0 {
Michael Yang's avatar
Michael Yang committed
398
				layer.MediaType = mediatype
399
400
				layers = append(layers, layer)
			}
Michael Yang's avatar
Michael Yang committed
401
402
403
		case "template", "system":
			fn(api.ProgressResponse{Status: fmt.Sprintf("creating %s layer", c.Name)})

Michael Yang's avatar
Michael Yang committed
404
			// remove duplicate layers
Michael Yang's avatar
Michael Yang committed
405
			layers = removeLayerFromLayers(layers, mediatype)
406

407
			layer, err := CreateLayer(strings.NewReader(c.Args))
408
			if err != nil {
409
				return err
410
			}
411

412
			if layer.Size > 0 {
Michael Yang's avatar
Michael Yang committed
413
				layer.MediaType = mediatype
414
415
				layers = append(layers, layer)
			}
416
		default:
417
			params[c.Name] = append(params[c.Name], c.Args)
418
419
420
		}
	}

Michael Yang's avatar
Michael Yang committed
421
	if len(params) > 0 {
Michael Yang's avatar
Michael Yang committed
422
		fn(api.ProgressResponse{Status: "creating parameters layer"})
Michael Yang's avatar
Michael Yang committed
423

424
		formattedParams, err := formatParams(params)
425
		if err != nil {
Michael Yang's avatar
Michael Yang committed
426
			return err
427
		}
428

Michael Yang's avatar
Michael Yang committed
429
		for k, v := range fromParams {
Michael Yang's avatar
Michael Yang committed
430
431
432
433
434
			if _, ok := formattedParams[k]; !ok {
				formattedParams[k] = v
			}
		}

Michael Yang's avatar
Michael Yang committed
435
		if config.ModelType == "65B" {
Michael Yang's avatar
Michael Yang committed
436
			if gqa, ok := formattedParams["gqa"].(int); ok && gqa == 8 {
Michael Yang's avatar
Michael Yang committed
437
438
439
440
				config.ModelType = "70B"
			}
		}

Michael Yang's avatar
Michael Yang committed
441
442
		var b bytes.Buffer
		if err := json.NewEncoder(&b).Encode(formattedParams); err != nil {
443
444
445
			return err
		}

Michael Yang's avatar
Michael Yang committed
446
447
		fn(api.ProgressResponse{Status: "creating config layer"})
		layer, err := CreateLayer(bytes.NewReader(b.Bytes()))
448
		if err != nil {
Michael Yang's avatar
Michael Yang committed
449
			return err
450
		}
Michael Yang's avatar
Michael Yang committed
451
452
453

		layer.MediaType = "application/vnd.ollama.image.params"
		layers = append(layers, layer)
454
455
	}

456
457
458
459
460
	digests, err := getLayerDigests(layers)
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
461
	configLayer, err := createConfigLayer(config, digests)
462
463
464
	if err != nil {
		return err
	}
Michael Yang's avatar
Michael Yang committed
465
466
467

	layers = append(layers, configLayer)
	delete(deleteMap, configLayer.Digest)
468

Michael Yang's avatar
Michael Yang committed
469
	if err := SaveLayers(layers, fn, false); err != nil {
470
471
472
		return err
	}

Michael Yang's avatar
Michael Yang committed
473
474
475
476
477
478
	var contentLayers []*Layer
	for _, layer := range layers {
		contentLayers = append(contentLayers, &layer.Layer)
		delete(deleteMap, layer.Digest)
	}

479
	fn(api.ProgressResponse{Status: "writing manifest"})
Michael Yang's avatar
Michael Yang committed
480
	if err := CreateManifest(name, configLayer, contentLayers); err != nil {
481
482
483
		return err
	}

Michael Yang's avatar
Michael Yang committed
484
485
	if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		if err := deleteUnusedLayers(nil, deleteMap, false); err != nil {
486
487
488
489
			return err
		}
	}

490
	fn(api.ProgressResponse{Status: "success"})
491
492
493
	return nil
}

Michael Yang's avatar
Michael Yang committed
494
func removeLayerFromLayers(layers []*LayerReader, mediaType string) []*LayerReader {
Michael Yang's avatar
Michael Yang committed
495
496
497
	return slices.DeleteFunc(layers, func(layer *LayerReader) bool {
		return layer.MediaType == mediaType
	})
498
499
}

500
func SaveLayers(layers []*LayerReader, fn func(resp api.ProgressResponse), force bool) error {
501
502
	// Write each of the layers to disk
	for _, layer := range layers {
Patrick Devine's avatar
Patrick Devine committed
503
		fp, err := GetBlobsPath(layer.Digest)
504
505
506
		if err != nil {
			return err
		}
507
508

		_, err = os.Stat(fp)
509
		if os.IsNotExist(err) || force {
510
511
			fn(api.ProgressResponse{Status: fmt.Sprintf("writing layer %s", layer.Digest)})

512
513
514
515
516
517
518
			out, err := os.Create(fp)
			if err != nil {
				log.Printf("couldn't create %s", fp)
				return err
			}
			defer out.Close()

Michael Yang's avatar
Michael Yang committed
519
			if _, err = io.Copy(out, layer.Reader); err != nil {
520
521
				return err
			}
Michael Yang's avatar
Michael Yang committed
522

523
		} else {
524
			fn(api.ProgressResponse{Status: fmt.Sprintf("using already created layer %s", layer.Digest)})
525
526
527
528
529
530
		}
	}

	return nil
}

Michael Yang's avatar
Michael Yang committed
531
func CreateManifest(name string, cfg *LayerReader, layers []*Layer) error {
532
	mp := ParseModelPath(name)
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
	manifest := ManifestV2{
		SchemaVersion: 2,
		MediaType:     "application/vnd.docker.distribution.manifest.v2+json",
		Config: Layer{
			MediaType: cfg.MediaType,
			Size:      cfg.Size,
			Digest:    cfg.Digest,
		},
		Layers: layers,
	}

	manifestJSON, err := json.Marshal(manifest)
	if err != nil {
		return err
	}

549
	fp, err := mp.GetManifestPath()
550
551
552
	if err != nil {
		return err
	}
553
554
555
	if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
		return err
	}
556
	return os.WriteFile(fp, manifestJSON, 0o644)
557
558
}

Michael Yang's avatar
Michael Yang committed
559
func GetLayerWithBufferFromLayer(layer *Layer) (*LayerReader, error) {
Patrick Devine's avatar
Patrick Devine committed
560
	fp, err := GetBlobsPath(layer.Digest)
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
	if err != nil {
		return nil, err
	}

	file, err := os.Open(fp)
	if err != nil {
		return nil, fmt.Errorf("could not open blob: %w", err)
	}
	defer file.Close()

	newLayer, err := CreateLayer(file)
	if err != nil {
		return nil, err
	}
	newLayer.MediaType = layer.MediaType
	return newLayer, nil
}

579
580
// formatParams converts specified parameter options to their correct types
func formatParams(params map[string][]string) (map[string]interface{}, error) {
581
582
583
	opts := api.Options{}
	valueOpts := reflect.ValueOf(&opts).Elem() // names of the fields in the options struct
	typeOpts := reflect.TypeOf(opts)           // types of the fields in the options struct
Michael Yang's avatar
Michael Yang committed
584

585
	// build map of json struct tags to their types
Michael Yang's avatar
Michael Yang committed
586
587
588
589
590
591
592
593
	jsonOpts := make(map[string]reflect.StructField)
	for _, field := range reflect.VisibleFields(typeOpts) {
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
		if jsonTag != "" {
			jsonOpts[jsonTag] = field
		}
	}

594
	out := make(map[string]interface{})
Michael Yang's avatar
Michael Yang committed
595
	// iterate params and set values based on json struct tags
596
	for key, vals := range params {
Michael Yang's avatar
Michael Yang committed
597
598
599
600
601
		if opt, ok := jsonOpts[key]; ok {
			field := valueOpts.FieldByName(opt.Name)
			if field.IsValid() && field.CanSet() {
				switch field.Kind() {
				case reflect.Float32:
602
					floatVal, err := strconv.ParseFloat(vals[0], 32)
Michael Yang's avatar
Michael Yang committed
603
					if err != nil {
604
						return nil, fmt.Errorf("invalid float value %s", vals)
Michael Yang's avatar
Michael Yang committed
605
606
					}

Michael Yang's avatar
Michael Yang committed
607
					out[key] = float32(floatVal)
Michael Yang's avatar
Michael Yang committed
608
				case reflect.Int:
Michael Yang's avatar
Michael Yang committed
609
					intVal, err := strconv.ParseInt(vals[0], 10, 64)
Michael Yang's avatar
Michael Yang committed
610
					if err != nil {
611
						return nil, fmt.Errorf("invalid int value %s", vals)
Michael Yang's avatar
Michael Yang committed
612
613
					}

614
					out[key] = intVal
Michael Yang's avatar
Michael Yang committed
615
				case reflect.Bool:
616
					boolVal, err := strconv.ParseBool(vals[0])
Michael Yang's avatar
Michael Yang committed
617
					if err != nil {
618
						return nil, fmt.Errorf("invalid bool value %s", vals)
Michael Yang's avatar
Michael Yang committed
619
620
					}

621
					out[key] = boolVal
Michael Yang's avatar
Michael Yang committed
622
				case reflect.String:
623
					out[key] = vals[0]
624
				case reflect.Slice:
625
626
					// TODO: only string slices are supported right now
					out[key] = vals
Michael Yang's avatar
Michael Yang committed
627
628
629
630
631
632
633
				default:
					return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
				}
			}
		}
	}

634
	return out, nil
635
636
}

Michael Yang's avatar
Michael Yang committed
637
func getLayerDigests(layers []*LayerReader) ([]string, error) {
638
639
640
641
642
643
644
645
646
647
648
	var digests []string
	for _, l := range layers {
		if l.Digest == "" {
			return nil, fmt.Errorf("layer is missing a digest")
		}
		digests = append(digests, l.Digest)
	}
	return digests, nil
}

// CreateLayer creates a Layer object from a given file
Michael Yang's avatar
Michael Yang committed
649
650
func CreateLayer(f io.ReadSeeker) (*LayerReader, error) {
	digest, size := GetSHA256Digest(f)
651
	f.Seek(0, io.SeekStart)
652

Michael Yang's avatar
Michael Yang committed
653
	layer := &LayerReader{
654
655
656
657
658
		Layer: Layer{
			MediaType: "application/vnd.docker.image.rootfs.diff.tar",
			Digest:    digest,
			Size:      size,
		},
Michael Yang's avatar
Michael Yang committed
659
		Reader: f,
660
661
662
663
664
	}

	return layer, nil
}

Patrick Devine's avatar
Patrick Devine committed
665
func CopyModel(src, dest string) error {
666
	srcModelPath := ParseModelPath(src)
667
	srcPath, err := srcModelPath.GetManifestPath()
668
669
670
671
	if err != nil {
		return err
	}

672
	destModelPath := ParseModelPath(dest)
673
	destPath, err := destModelPath.GetManifestPath()
Patrick Devine's avatar
Patrick Devine committed
674
675
676
	if err != nil {
		return err
	}
677
678
679
	if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
		return err
	}
Patrick Devine's avatar
Patrick Devine committed
680
681

	// copy the file
Michael Yang's avatar
Michael Yang committed
682
	input, err := os.ReadFile(srcPath)
Patrick Devine's avatar
Patrick Devine committed
683
684
685
686
687
	if err != nil {
		fmt.Println("Error reading file:", err)
		return err
	}

Michael Yang's avatar
Michael Yang committed
688
	err = os.WriteFile(destPath, input, 0o644)
Patrick Devine's avatar
Patrick Devine committed
689
690
691
692
693
694
695
696
	if err != nil {
		fmt.Println("Error reading file:", err)
		return err
	}

	return nil
}

Michael Yang's avatar
Michael Yang committed
697
func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]struct{}, dryRun bool) error {
698
699
700
701
	fp, err := GetManifestPath()
	if err != nil {
		return err
	}
Michael Yang's avatar
Michael Yang committed
702
703
704
705

	walkFunc := func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			return nil
706
707
		}

Michael Yang's avatar
Michael Yang committed
708
709
710
711
		dir, file := filepath.Split(path)
		dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
		tag := strings.Join([]string{dir, file}, ":")
		fmp := ParseModelPath(tag)
712

Michael Yang's avatar
Michael Yang committed
713
		// skip the manifest we're trying to delete
714
		if skipModelPath != nil && skipModelPath.GetFullTagname() == fmp.GetFullTagname() {
Michael Yang's avatar
Michael Yang committed
715
			return nil
716
		}
Michael Yang's avatar
Michael Yang committed
717
718
719
720
721
722
723
724
725
726
727
728

		// save (i.e. delete from the deleteMap) any files used in other manifests
		manifest, _, err := GetManifest(fmp)
		if err != nil {
			return nil
		}

		for _, layer := range manifest.Layers {
			delete(deleteMap, layer.Digest)
		}

		delete(deleteMap, manifest.Config.Digest)
729
		return nil
Michael Yang's avatar
Michael Yang committed
730
731
732
	}

	if err := filepath.Walk(fp, walkFunc); err != nil {
Michael Yang's avatar
Michael Yang committed
733
734
		return err
	}
735
736

	// only delete the files which are still in the deleteMap
Michael Yang's avatar
Michael Yang committed
737
738
739
740
741
742
743
744
745
	for k := range deleteMap {
		fp, err := GetBlobsPath(k)
		if err != nil {
			log.Printf("couldn't get file path for '%s': %v", k, err)
			continue
		}
		if !dryRun {
			if err := os.Remove(fp); err != nil {
				log.Printf("couldn't remove file '%s': %v", fp, err)
746
747
				continue
			}
Michael Yang's avatar
Michael Yang committed
748
749
		} else {
			log.Printf("wanted to remove: %s", fp)
750
751
752
		}
	}

753
754
755
756
	return nil
}

func PruneLayers() error {
Michael Yang's avatar
Michael Yang committed
757
	deleteMap := make(map[string]struct{})
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
	p, err := GetBlobsPath("")
	if err != nil {
		return err
	}

	blobs, err := os.ReadDir(p)
	if err != nil {
		log.Printf("couldn't read dir '%s': %v", p, err)
		return err
	}

	for _, blob := range blobs {
		name := blob.Name()
		if runtime.GOOS == "windows" {
			name = strings.ReplaceAll(name, "-", ":")
		}
Michael Yang's avatar
Michael Yang committed
774
775
776
		if strings.HasPrefix(name, "sha256:") {
			deleteMap[name] = struct{}{}
		}
777
778
779
780
781
782
783
784
785
786
787
788
789
790
	}

	log.Printf("total blobs: %d", len(deleteMap))

	err = deleteUnusedLayers(nil, deleteMap, false)
	if err != nil {
		return err
	}

	log.Printf("total unused blobs removed: %d", len(deleteMap))

	return nil
}

Michael Yang's avatar
Michael Yang committed
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
func PruneDirectory(path string) error {
	info, err := os.Lstat(path)
	if err != nil {
		return err
	}

	if info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
		entries, err := os.ReadDir(path)
		if err != nil {
			return err
		}

		for _, entry := range entries {
			if err := PruneDirectory(filepath.Join(path, entry.Name())); err != nil {
				return err
			}
		}

		entries, err = os.ReadDir(path)
		if err != nil {
			return err
		}

		if len(entries) > 0 {
			return nil
		}

		return os.Remove(path)
	}

	return nil
}

824
825
826
827
828
829
830
func DeleteModel(name string) error {
	mp := ParseModelPath(name)
	manifest, _, err := GetManifest(mp)
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
831
	deleteMap := make(map[string]struct{})
832
	for _, layer := range manifest.Layers {
Michael Yang's avatar
Michael Yang committed
833
		deleteMap[layer.Digest] = struct{}{}
834
	}
Michael Yang's avatar
Michael Yang committed
835
	deleteMap[manifest.Config.Digest] = struct{}{}
836
837
838
839
840
841

	err = deleteUnusedLayers(&mp, deleteMap, false)
	if err != nil {
		return err
	}

842
	fp, err := mp.GetManifestPath()
843
844
845
846
847
848
849
850
851
852
853
854
	if err != nil {
		return err
	}
	err = os.Remove(fp)
	if err != nil {
		log.Printf("couldn't remove manifest file '%s': %v", fp, err)
		return err
	}

	return nil
}

Patrick Devine's avatar
Patrick Devine committed
855
func ShowModelfile(model *Model) (string, error) {
Michael Yang's avatar
Michael Yang committed
856
	var mt struct {
Patrick Devine's avatar
Patrick Devine committed
857
		*Model
Michael Yang's avatar
Michael Yang committed
858
		From       string
Michael Yang's avatar
Michael Yang committed
859
		Parameters map[string][]any
Patrick Devine's avatar
Patrick Devine committed
860
861
	}

Michael Yang's avatar
Michael Yang committed
862
	mt.Parameters = make(map[string][]any)
Patrick Devine's avatar
Patrick Devine committed
863
	for k, v := range model.Options {
Michael Yang's avatar
Michael Yang committed
864
865
866
		if s, ok := v.([]any); ok {
			mt.Parameters[k] = s
			continue
Patrick Devine's avatar
Patrick Devine committed
867
868
		}

Michael Yang's avatar
Michael Yang committed
869
		mt.Parameters[k] = []any{v}
Patrick Devine's avatar
Patrick Devine committed
870
871
	}

Michael Yang's avatar
Michael Yang committed
872
873
	mt.Model = model
	mt.From = model.ModelPath
Patrick Devine's avatar
Patrick Devine committed
874

Michael Yang's avatar
Michael Yang committed
875
	if model.OriginalModel != "" {
Daniel Reis's avatar
Daniel Reis committed
876
		mt.From = model.OriginalModel
Patrick Devine's avatar
Patrick Devine committed
877
878
879
880
881
882
883
884
	}

	modelFile := `# Modelfile generated by "ollama show"
# To build a new Modelfile based on this one, replace the FROM line with:
# FROM {{ .ShortName }}

FROM {{ .From }}
TEMPLATE """{{ .Template }}"""
885
886

{{- if .System }}
Patrick Devine's avatar
Patrick Devine committed
887
SYSTEM """{{ .System }}"""
888
{{- end }}
889
890
891
892

{{- range $adapter := .AdapterPaths }}
ADAPTER {{ $adapter }}
{{- end }}
Michael Yang's avatar
Michael Yang committed
893

Michael Yang's avatar
Michael Yang committed
894
895
896
897
{{- range $k, $v := .Parameters }}
{{- range $parameter := $v }}
PARAMETER {{ $k }} {{ printf "%#v" $parameter }}
{{- end }}
Michael Yang's avatar
Michael Yang committed
898
{{- end }}`
Patrick Devine's avatar
Patrick Devine committed
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915

	tmpl, err := template.New("").Parse(modelFile)
	if err != nil {
		log.Printf("error parsing template: %q", err)
		return "", err
	}

	var buf bytes.Buffer

	if err = tmpl.Execute(&buf, mt); err != nil {
		log.Printf("error executing template: %q", err)
		return "", err
	}

	return buf.String(), nil
}

916
func PushModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
917
	mp := ParseModelPath(name)
918
919
	fn(api.ProgressResponse{Status: "retrieving manifest"})

920
921
922
923
	if mp.ProtocolScheme == "http" && !regOpts.Insecure {
		return fmt.Errorf("insecure protocol http")
	}

Patrick Devine's avatar
Patrick Devine committed
924
	manifest, _, err := GetManifest(mp)
925
	if err != nil {
926
		fn(api.ProgressResponse{Status: "couldn't retrieve manifest"})
927
928
929
930
		return err
	}

	var layers []*Layer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
931
	layers = append(layers, manifest.Layers...)
932
933
934
	layers = append(layers, &manifest.Config)

	for _, layer := range layers {
Michael Yang's avatar
Michael Yang committed
935
		if err := uploadBlob(ctx, mp, layer, regOpts, fn); err != nil {
936
			log.Printf("error uploading blob: %v", err)
937
938
939
			if errors.Is(err, errUnauthorized) {
				return fmt.Errorf("unable to push %s, make sure this namespace exists and you are authorized to push to it", ParseModelPath(name).GetNamespaceRepository())
			}
940
941
			return err
		}
942
943
	}

944
	fn(api.ProgressResponse{Status: "pushing manifest"})
Michael Yang's avatar
Michael Yang committed
945
946
	requestURL := mp.BaseURL()
	requestURL = requestURL.JoinPath("v2", mp.GetNamespaceRepository(), "manifests", mp.Tag)
947
948
949
950
951
952

	manifestJSON, err := json.Marshal(manifest)
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
953
954
	headers := make(http.Header)
	headers.Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
Michael Yang's avatar
Michael Yang committed
955
	resp, err := makeRequestWithRetry(ctx, http.MethodPut, requestURL, headers, bytes.NewReader(manifestJSON), regOpts)
956
957
958
959
960
	if err != nil {
		return err
	}
	defer resp.Body.Close()

961
	fn(api.ProgressResponse{Status: "success"})
962
963
964
965

	return nil
}

966
func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
967
968
	mp := ParseModelPath(name)

969
970
971
972
973
	var manifest *ManifestV2
	var err error
	var noprune string

	// build deleteMap to prune unused layers
Michael Yang's avatar
Michael Yang committed
974
	deleteMap := make(map[string]struct{})
975
976
977
978
979
980
981
982
983

	if noprune = os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		manifest, _, err = GetManifest(mp)
		if err != nil && !errors.Is(err, os.ErrNotExist) {
			return err
		}

		if manifest != nil {
			for _, l := range manifest.Layers {
Michael Yang's avatar
Michael Yang committed
984
				deleteMap[l.Digest] = struct{}{}
985
			}
Michael Yang's avatar
Michael Yang committed
986
			deleteMap[manifest.Config.Digest] = struct{}{}
987
988
989
		}
	}

990
991
	if mp.ProtocolScheme == "http" && !regOpts.Insecure {
		return fmt.Errorf("insecure protocol http")
992
	}
993

994
	fn(api.ProgressResponse{Status: "pulling manifest"})
995

996
	manifest, err = pullModelManifest(ctx, mp, regOpts)
997
	if err != nil {
998
		return fmt.Errorf("pull model manifest: %s", err)
999
1000
1001
	}

	var layers []*Layer
Bruce MacDonald's avatar
Bruce MacDonald committed
1002
	layers = append(layers, manifest.Layers...)
1003
1004
1005
	layers = append(layers, &manifest.Config)

	for _, layer := range layers {
1006
1007
1008
1009
1010
1011
1012
1013
		if err := downloadBlob(
			ctx,
			downloadOpts{
				mp:      mp,
				digest:  layer.Digest,
				regOpts: regOpts,
				fn:      fn,
			}); err != nil {
1014
1015
			return err
		}
1016
		delete(deleteMap, layer.Digest)
1017
	}
1018
	delete(deleteMap, manifest.Config.Digest)
1019

Michael Yang's avatar
Michael Yang committed
1020
1021
1022
	fn(api.ProgressResponse{Status: "verifying sha256 digest"})
	for _, layer := range layers {
		if err := verifyBlob(layer.Digest); err != nil {
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
			if errors.Is(err, errDigestMismatch) {
				// something went wrong, delete the blob
				fp, err := GetBlobsPath(layer.Digest)
				if err != nil {
					return err
				}
				if err := os.Remove(fp); err != nil {
					// log this, but return the original error
					log.Printf("couldn't remove file with digest mismatch '%s': %v", fp, err)
				}
			}
Michael Yang's avatar
Michael Yang committed
1034
1035
1036
1037
			return err
		}
	}

1038
	fn(api.ProgressResponse{Status: "writing manifest"})
1039

1040
	manifestJSON, err := json.Marshal(manifest)
1041
1042
1043
1044
	if err != nil {
		return err
	}

1045
	fp, err := mp.GetManifestPath()
1046
1047
1048
	if err != nil {
		return err
	}
1049
1050
1051
	if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
		return err
	}
1052

Bruce MacDonald's avatar
Bruce MacDonald committed
1053
	err = os.WriteFile(fp, manifestJSON, 0o644)
1054
1055
1056
1057
1058
	if err != nil {
		log.Printf("couldn't write to %s", fp)
		return err
	}

1059
1060
1061
1062
1063
1064
1065
1066
	if noprune == "" {
		fn(api.ProgressResponse{Status: "removing any unused layers"})
		err = deleteUnusedLayers(nil, deleteMap, false)
		if err != nil {
			return err
		}
	}

1067
	fn(api.ProgressResponse{Status: "success"})
1068
1069
1070
1071

	return nil
}

1072
func pullModelManifest(ctx context.Context, mp ModelPath, regOpts *RegistryOptions) (*ManifestV2, error) {
Michael Yang's avatar
Michael Yang committed
1073
	requestURL := mp.BaseURL().JoinPath("v2", mp.GetNamespaceRepository(), "manifests", mp.Tag)
1074

Michael Yang's avatar
Michael Yang committed
1075
1076
	headers := make(http.Header)
	headers.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json")
Michael Yang's avatar
Michael Yang committed
1077
	resp, err := makeRequestWithRetry(ctx, http.MethodGet, requestURL, headers, nil, regOpts)
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var m *ManifestV2
	if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
		return nil, err
	}

	return m, err
}

1091
1092
1093
1094
func createConfigLayer(config ConfigV2, layers []string) (*LayerReader, error) {
	config.RootFS = RootFS{
		Type:    "layers",
		DiffIDs: layers,
1095
1096
1097
1098
1099
1100
1101
	}

	configJSON, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

1102
	digest, size := GetSHA256Digest(bytes.NewBuffer(configJSON))
1103

Michael Yang's avatar
Michael Yang committed
1104
	layer := &LayerReader{
1105
1106
1107
1108
1109
		Layer: Layer{
			MediaType: "application/vnd.docker.container.image.v1+json",
			Digest:    digest,
			Size:      size,
		},
1110
		Reader: bytes.NewBuffer(configJSON),
1111
1112
1113
1114
1115
	}
	return layer, nil
}

// GetSHA256Digest returns the SHA256 hash of a given buffer and returns it, and the size of buffer
Michael Yang's avatar
Michael Yang committed
1116
func GetSHA256Digest(r io.Reader) (string, int64) {
Michael Yang's avatar
Michael Yang committed
1117
1118
1119
1120
1121
1122
	h := sha256.New()
	n, err := io.Copy(h, r)
	if err != nil {
		log.Fatal(err)
	}

Michael Yang's avatar
Michael Yang committed
1123
	return fmt.Sprintf("sha256:%x", h.Sum(nil)), n
1124
1125
}

1126
1127
var errUnauthorized = fmt.Errorf("unauthorized")

Michael Yang's avatar
Michael Yang committed
1128
func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.ReadSeeker, regOpts *RegistryOptions) (*http.Response, error) {
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
	resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts)
	if err != nil {
		if !errors.Is(err, context.Canceled) {
			log.Printf("request failed: %v", err)
		}
		return nil, err
	}

	switch {
	case resp.StatusCode == http.StatusUnauthorized:
		// Handle authentication error with one retry
		auth := resp.Header.Get("www-authenticate")
		authRedir := ParseAuthRedirectString(auth)
		token, err := getAuthToken(ctx, authRedir)
Michael Yang's avatar
Michael Yang committed
1143
1144
1145
		if err != nil {
			return nil, err
		}
1146
1147
1148
		regOpts.Token = token
		if body != nil {
			_, err = body.Seek(0, io.SeekStart)
Michael Yang's avatar
Michael Yang committed
1149
1150
1151
1152
			if err != nil {
				return nil, err
			}
		}
1153
1154
1155
1156
1157
1158
1159
1160
1161
		return makeRequest(ctx, method, requestURL, headers, body, regOpts)
	case resp.StatusCode == http.StatusNotFound:
		return nil, os.ErrNotExist
	case resp.StatusCode >= http.StatusBadRequest:
		responseBody, err := io.ReadAll(resp.Body)
		if err != nil {
			return nil, fmt.Errorf("%d: %s", resp.StatusCode, err)
		}
		return nil, fmt.Errorf("%d: %s", resp.StatusCode, responseBody)
Michael Yang's avatar
Michael Yang committed
1162
1163
	}

1164
	return resp, nil
Michael Yang's avatar
Michael Yang committed
1165
1166
}

Michael Yang's avatar
Michael Yang committed
1167
func makeRequest(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.Reader, regOpts *RegistryOptions) (*http.Response, error) {
Michael Yang's avatar
Michael Yang committed
1168
	if requestURL.Scheme != "http" && regOpts != nil && regOpts.Insecure {
Michael Yang's avatar
Michael Yang committed
1169
		requestURL.Scheme = "http"
1170
1171
	}

Michael Yang's avatar
Michael Yang committed
1172
	req, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
1173
1174
1175
1176
	if err != nil {
		return nil, err
	}

Michael Yang's avatar
Michael Yang committed
1177
1178
1179
1180
	if headers != nil {
		req.Header = headers
	}

Michael Yang's avatar
Michael Yang committed
1181
1182
1183
1184
1185
1186
	if regOpts != nil {
		if regOpts.Token != "" {
			req.Header.Set("Authorization", "Bearer "+regOpts.Token)
		} else if regOpts.Username != "" && regOpts.Password != "" {
			req.SetBasicAuth(regOpts.Username, regOpts.Password)
		}
1187
1188
	}

Michael Yang's avatar
Michael Yang committed
1189
	req.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
1190

Michael Yang's avatar
Michael Yang committed
1191
1192
1193
1194
1195
1196
1197
1198
1199
	if s := req.Header.Get("Content-Length"); s != "" {
		contentLength, err := strconv.ParseInt(s, 10, 64)
		if err != nil {
			return nil, err
		}

		req.ContentLength = contentLength
	}

Michael Yang's avatar
Michael Yang committed
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
	proxyURL, err := http.ProxyFromEnvironment(req)
	if err != nil {
		return nil, err
	}

	client := http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyURL(proxyURL),
		},
	}

	resp, err := client.Do(req)
1212
1213
1214
1215
1216
1217
	if err != nil {
		return nil, err
	}

	return resp, nil
}
Michael Yang's avatar
Michael Yang committed
1218

Patrick Devine's avatar
Patrick Devine committed
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
func getValue(header, key string) string {
	startIdx := strings.Index(header, key+"=")
	if startIdx == -1 {
		return ""
	}

	// Move the index to the starting quote after the key.
	startIdx += len(key) + 2
	endIdx := startIdx

	for endIdx < len(header) {
		if header[endIdx] == '"' {
			if endIdx+1 < len(header) && header[endIdx+1] != ',' { // If the next character isn't a comma, continue
				endIdx++
				continue
			}
			break
		}
		endIdx++
	}
	return header[startIdx:endIdx]
}

func ParseAuthRedirectString(authStr string) AuthRedirect {
	authStr = strings.TrimPrefix(authStr, "Bearer ")

	return AuthRedirect{
		Realm:   getValue(authStr, "realm"),
		Service: getValue(authStr, "service"),
		Scope:   getValue(authStr, "scope"),
	}
}

1252
1253
var errDigestMismatch = fmt.Errorf("digest mismatch, file must be downloaded again")

Michael Yang's avatar
Michael Yang committed
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
func verifyBlob(digest string) error {
	fp, err := GetBlobsPath(digest)
	if err != nil {
		return err
	}

	f, err := os.Open(fp)
	if err != nil {
		return err
	}
	defer f.Close()

	fileDigest, _ := GetSHA256Digest(f)
	if digest != fileDigest {
1268
		return fmt.Errorf("%w: want %s, got %s", errDigestMismatch, digest, fileDigest)
Michael Yang's avatar
Michael Yang committed
1269
1270
1271
1272
	}

	return nil
}