images.go 30.2 KB
Newer Older
1
2
3
4
package server

import (
	"bytes"
5
	"context"
6
	"crypto/sha256"
Patrick Devine's avatar
Patrick Devine committed
7
	"encoding/hex"
8
9
10
11
12
13
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
Michael Yang's avatar
Michael Yang committed
14
	"net/url"
15
16
	"os"
	"path/filepath"
Michael Yang's avatar
Michael Yang committed
17
	"reflect"
Michael Yang's avatar
Michael Yang committed
18
	"runtime"
19
20
	"strconv"
	"strings"
Quinn Slack's avatar
Quinn Slack committed
21
	"text/template"
22

Michael Yang's avatar
Michael Yang committed
23
24
	"golang.org/x/exp/slices"

25
	"github.com/jmorganca/ollama/api"
26
	"github.com/jmorganca/ollama/llm"
27
	"github.com/jmorganca/ollama/parser"
Michael Yang's avatar
Michael Yang committed
28
	"github.com/jmorganca/ollama/version"
29
30
)

31
32
33
34
type RegistryOptions struct {
	Insecure bool
	Username string
	Password string
Patrick Devine's avatar
Patrick Devine committed
35
	Token    string
36
37
}

38
type Model struct {
Patrick Devine's avatar
Patrick Devine committed
39
40
41
42
43
44
45
46
47
48
	Name          string `json:"name"`
	ShortName     string
	ModelPath     string
	OriginalModel string
	AdapterPaths  []string
	Template      string
	System        string
	License       []string
	Digest        string
	Options       map[string]interface{}
49
50
}

51
func (m *Model) Prompt(request api.GenerateRequest) (string, error) {
52
53
54
55
56
57
	t := m.Template
	if request.Template != "" {
		t = request.Template
	}

	tmpl, err := template.New("").Parse(t)
58
59
60
61
62
	if err != nil {
		return "", err
	}

	var vars struct {
Michael Yang's avatar
Michael Yang committed
63
		First  bool
64
65
66
67
		System string
		Prompt string
	}

Michael Yang's avatar
Michael Yang committed
68
	vars.First = len(request.Context) == 0
69
	vars.System = m.System
70
71
	vars.Prompt = request.Prompt

72
73
74
75
	if request.System != "" {
		vars.System = request.System
	}

76
77
78
79
80
81
82
83
	var sb strings.Builder
	if err := tmpl.Execute(&sb, vars); err != nil {
		return "", err
	}

	return sb.String(), nil
}

84
85
86
87
88
89
90
91
92
93
type ManifestV2 struct {
	SchemaVersion int      `json:"schemaVersion"`
	MediaType     string   `json:"mediaType"`
	Config        Layer    `json:"config"`
	Layers        []*Layer `json:"layers"`
}

type Layer struct {
	MediaType string `json:"mediaType"`
	Digest    string `json:"digest"`
Michael Yang's avatar
Michael Yang committed
94
	Size      int64  `json:"size"`
Michael Yang's avatar
Michael Yang committed
95
	From      string `json:"from,omitempty"`
96
97
}

Michael Yang's avatar
Michael Yang committed
98
type LayerReader struct {
99
	Layer
Michael Yang's avatar
Michael Yang committed
100
	io.Reader
101
102
103
}

type ConfigV2 struct {
Michael Yang's avatar
Michael Yang committed
104
105
106
107
108
	ModelFormat string `json:"model_format"`
	ModelFamily string `json:"model_family"`
	ModelType   string `json:"model_type"`
	FileType    string `json:"file_type"`
	RootFS      RootFS `json:"rootfs"`
109
110

	// required by spec
111
112
113
114
115
116
117
118
119
	Architecture string `json:"architecture"`
	OS           string `json:"os"`
}

type RootFS struct {
	Type    string   `json:"type"`
	DiffIDs []string `json:"diff_ids"`
}

Michael Yang's avatar
Michael Yang committed
120
func (m *ManifestV2) GetTotalSize() (total int64) {
Patrick Devine's avatar
Patrick Devine committed
121
122
123
	for _, layer := range m.Layers {
		total += layer.Size
	}
Michael Yang's avatar
Michael Yang committed
124

Patrick Devine's avatar
Patrick Devine committed
125
126
127
128
	total += m.Config.Size
	return total
}

Patrick Devine's avatar
Patrick Devine committed
129
func GetManifest(mp ModelPath) (*ManifestV2, string, error) {
130
	fp, err := mp.GetManifestPath()
131
	if err != nil {
Patrick Devine's avatar
Patrick Devine committed
132
		return nil, "", err
133
	}
134

135
	if _, err = os.Stat(fp); err != nil {
Patrick Devine's avatar
Patrick Devine committed
136
		return nil, "", err
137
138
139
140
	}

	var manifest *ManifestV2

141
	bts, err := os.ReadFile(fp)
142
	if err != nil {
Patrick Devine's avatar
Patrick Devine committed
143
		return nil, "", fmt.Errorf("couldn't open file '%s'", fp)
144
145
	}

Patrick Devine's avatar
Patrick Devine committed
146
147
148
	shaSum := sha256.Sum256(bts)
	shaStr := hex.EncodeToString(shaSum[:])

149
	if err := json.Unmarshal(bts, &manifest); err != nil {
Patrick Devine's avatar
Patrick Devine committed
150
		return nil, "", err
151
152
	}

Patrick Devine's avatar
Patrick Devine committed
153
	return manifest, shaStr, nil
154
155
156
}

func GetModel(name string) (*Model, error) {
157
	mp := ParseModelPath(name)
Patrick Devine's avatar
Patrick Devine committed
158
	manifest, digest, err := GetManifest(mp)
159
160
161
162
163
	if err != nil {
		return nil, err
	}

	model := &Model{
164
165
166
167
168
		Name:      mp.GetFullTagname(),
		ShortName: mp.GetShortTagname(),
		Digest:    digest,
		Template:  "{{ .Prompt }}",
		License:   []string{},
169
170
171
	}

	for _, layer := range manifest.Layers {
Patrick Devine's avatar
Patrick Devine committed
172
		filename, err := GetBlobsPath(layer.Digest)
173
174
175
176
		if err != nil {
			return nil, err
		}

177
178
179
		switch layer.MediaType {
		case "application/vnd.ollama.image.model":
			model.ModelPath = filename
Patrick Devine's avatar
Patrick Devine committed
180
			model.OriginalModel = layer.From
181
		case "application/vnd.ollama.image.embed":
182
183
184
			// Deprecated in versions  > 0.1.2
			// TODO: remove this warning in a future version
			log.Print("WARNING: model contains embeddings, but embeddings in modelfiles have been deprecated and will be ignored.")
185
186
		case "application/vnd.ollama.image.adapter":
			model.AdapterPaths = append(model.AdapterPaths, filename)
187
188
189
190
191
192
193
194
195
		case "application/vnd.ollama.image.template":
			bts, err := os.ReadFile(filename)
			if err != nil {
				return nil, err
			}

			model.Template = string(bts)
		case "application/vnd.ollama.image.system":
			bts, err := os.ReadFile(filename)
196
197
198
			if err != nil {
				return nil, err
			}
199
200

			model.System = string(bts)
201
202
203
204
205
206
207
		case "application/vnd.ollama.image.prompt":
			bts, err := os.ReadFile(filename)
			if err != nil {
				return nil, err
			}

			model.Template = string(bts)
208
		case "application/vnd.ollama.image.params":
Michael Yang's avatar
Michael Yang committed
209
210
211
212
213
			params, err := os.Open(filename)
			if err != nil {
				return nil, err
			}
			defer params.Close()
214

215
			// parse model options parameters into a map so that we can see which fields have been specified explicitly
216
			if err = json.NewDecoder(params).Decode(&model.Options); err != nil {
217
218
				return nil, err
			}
Patrick Devine's avatar
Patrick Devine committed
219
220
221
222
223
224
		case "application/vnd.ollama.image.license":
			bts, err := os.ReadFile(filename)
			if err != nil {
				return nil, err
			}
			model.License = append(model.License, string(bts))
225
226
227
228
229
230
		}
	}

	return model, nil
}

231
232
func filenameWithPath(path, f string) (string, error) {
	// if filePath starts with ~/, replace it with the user's home directory.
Michael Yang's avatar
Michael Yang committed
233
234
	if strings.HasPrefix(f, fmt.Sprintf("~%s", string(os.PathSeparator))) {
		parts := strings.Split(f, string(os.PathSeparator))
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
		home, err := os.UserHomeDir()
		if err != nil {
			return "", fmt.Errorf("failed to open file: %v", err)
		}

		f = filepath.Join(home, filepath.Join(parts[1:]...))
	}

	// if filePath is not an absolute path, make it relative to the modelfile path
	if !filepath.IsAbs(f) {
		f = filepath.Join(filepath.Dir(path), f)
	}

	return f, nil
}

251
func CreateModel(ctx context.Context, name string, path string, fn func(resp api.ProgressResponse)) error {
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
	mp := ParseModelPath(name)

	var manifest *ManifestV2
	var err error
	var noprune string

	// build deleteMap to prune unused layers
	deleteMap := make(map[string]bool)

	if noprune = os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		manifest, _, err = GetManifest(mp)
		if err != nil && !errors.Is(err, os.ErrNotExist) {
			return err
		}

		if manifest != nil {
			for _, l := range manifest.Layers {
				deleteMap[l.Digest] = true
			}
			deleteMap[manifest.Config.Digest] = true
		}
	}

275
276
	mf, err := os.Open(path)
	if err != nil {
277
		fn(api.ProgressResponse{Status: fmt.Sprintf("couldn't open modelfile '%s'", path)})
278
		return fmt.Errorf("failed to open file: %w", err)
279
	}
280
	defer mf.Close()
281

282
	fn(api.ProgressResponse{Status: "parsing modelfile"})
283
284
285
286
287
	commands, err := parser.Parse(mf)
	if err != nil {
		return err
	}

288
289
290
291
292
	config := ConfigV2{
		Architecture: "amd64",
		OS:           "linux",
	}

Michael Yang's avatar
Michael Yang committed
293
	var layers []*LayerReader
294
	params := make(map[string][]string)
Michael Yang's avatar
Michael Yang committed
295
	var sourceParams map[string]any
296
	for _, c := range commands {
297
		log.Printf("[%s] - %s\n", c.Name, c.Args)
298
299
		switch c.Name {
		case "model":
300
			fn(api.ProgressResponse{Status: "looking for model"})
301

302
			mp := ParseModelPath(c.Args)
Patrick Devine's avatar
Patrick Devine committed
303
			mf, _, err := GetManifest(mp)
304
			if err != nil {
305
306
307
				modelFile, err := filenameWithPath(path, c.Args)
				if err != nil {
					return err
308
				}
309
				if _, err := os.Stat(modelFile); err != nil {
310
311
312
					// the model file does not exist, try pulling it
					if errors.Is(err, os.ErrNotExist) {
						fn(api.ProgressResponse{Status: "pulling model file"})
313
						if err := PullModel(ctx, c.Args, &RegistryOptions{}, fn); err != nil {
314
315
							return err
						}
Patrick Devine's avatar
Patrick Devine committed
316
						mf, _, err = GetManifest(mp)
317
318
319
320
321
322
323
324
325
						if err != nil {
							return fmt.Errorf("failed to open file after pull: %v", err)
						}
					} else {
						return err
					}
				} else {
					// create a model from this specified file
					fn(api.ProgressResponse{Status: "creating model layer"})
326
					file, err := os.Open(modelFile)
327
328
329
330
331
					if err != nil {
						return fmt.Errorf("failed to open file: %v", err)
					}
					defer file.Close()

Bruce MacDonald's avatar
Bruce MacDonald committed
332
					ggml, err := llm.DecodeGGML(file)
333
334
335
336
					if err != nil {
						return err
					}

337
					config.ModelFormat = ggml.Name()
Michael Yang's avatar
Michael Yang committed
338
339
340
					config.ModelFamily = ggml.ModelFamily()
					config.ModelType = ggml.ModelType()
					config.FileType = ggml.FileType()
341
342
343
344

					// reset the file
					file.Seek(0, io.SeekStart)

345
346
347
348
349
350
					l, err := CreateLayer(file)
					if err != nil {
						return fmt.Errorf("failed to create layer: %v", err)
					}
					l.MediaType = "application/vnd.ollama.image.model"
					layers = append(layers, l)
351
				}
352
			}
353

354
			if mf != nil {
355
				fn(api.ProgressResponse{Status: "reading model metadata"})
Michael Yang's avatar
Michael Yang committed
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
				sourceBlobPath, err := GetBlobsPath(mf.Config.Digest)
				if err != nil {
					return err
				}

				sourceBlob, err := os.Open(sourceBlobPath)
				if err != nil {
					return err
				}
				defer sourceBlob.Close()

				var source ConfigV2
				if err := json.NewDecoder(sourceBlob).Decode(&source); err != nil {
					return err
				}

372
				// copy the model metadata
Michael Yang's avatar
Michael Yang committed
373
374
				config.ModelFamily = source.ModelFamily
				config.ModelType = source.ModelType
375
				config.ModelFormat = source.ModelFormat
Michael Yang's avatar
Michael Yang committed
376
377
				config.FileType = source.FileType

378
				for _, l := range mf.Layers {
Michael Yang's avatar
Michael Yang committed
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
					if l.MediaType == "application/vnd.ollama.image.params" {
						sourceParamsBlobPath, err := GetBlobsPath(l.Digest)
						if err != nil {
							return err
						}

						sourceParamsBlob, err := os.Open(sourceParamsBlobPath)
						if err != nil {
							return err
						}
						defer sourceParamsBlob.Close()

						if err := json.NewDecoder(sourceParamsBlob).Decode(&sourceParams); err != nil {
							return err
						}
					}

396
397
398
399
					newLayer, err := GetLayerWithBufferFromLayer(l)
					if err != nil {
						return err
					}
Michael Yang's avatar
Michael Yang committed
400
					newLayer.From = mp.GetNamespaceRepository()
401
402
403
					layers = append(layers, newLayer)
				}
			}
404
405
406
		case "adapter":
			fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)})

Michael Yang's avatar
Michael Yang committed
407
408
409
			fp, err := filenameWithPath(path, c.Args)
			if err != nil {
				return err
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
			}

			// create a model from this specified file
			fn(api.ProgressResponse{Status: "creating model layer"})

			file, err := os.Open(fp)
			if err != nil {
				return fmt.Errorf("failed to open file: %v", err)
			}
			defer file.Close()

			l, err := CreateLayer(file)
			if err != nil {
				return fmt.Errorf("failed to create layer: %v", err)
			}
			l.MediaType = "application/vnd.ollama.image.adapter"
			layers = append(layers, l)
Bruce MacDonald's avatar
Bruce MacDonald committed
427
428
429
430
431
432
433
434
435
		case "license":
			fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)})
			mediaType := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name)

			layer, err := CreateLayer(strings.NewReader(c.Args))
			if err != nil {
				return err
			}

436
437
438
439
			if layer.Size > 0 {
				layer.MediaType = mediaType
				layers = append(layers, layer)
			}
Bruce MacDonald's avatar
Bruce MacDonald committed
440
		case "template", "system", "prompt":
441
			fn(api.ProgressResponse{Status: fmt.Sprintf("creating model %s layer", c.Name)})
442
			// remove the layer if one exists
443
444
			mediaType := fmt.Sprintf("application/vnd.ollama.image.%s", c.Name)
			layers = removeLayerFromLayers(layers, mediaType)
445

446
			layer, err := CreateLayer(strings.NewReader(c.Args))
447
			if err != nil {
448
				return err
449
			}
450

451
452
453
454
			if layer.Size > 0 {
				layer.MediaType = mediaType
				layers = append(layers, layer)
			}
455
		default:
456
			// runtime parameters, build a list of args for each parameter to allow multiple values to be specified (ex: multiple stop sequences)
457
			params[c.Name] = append(params[c.Name], c.Args)
458
459
460
461
		}
	}

	// Create a single layer for the parameters
Michael Yang's avatar
Michael Yang committed
462
	if len(params) > 0 {
463
		fn(api.ProgressResponse{Status: "creating parameter layer"})
Michael Yang's avatar
Michael Yang committed
464

465
		layers = removeLayerFromLayers(layers, "application/vnd.ollama.image.params")
466
		formattedParams, err := formatParams(params)
467
468
469
		if err != nil {
			return fmt.Errorf("couldn't create params json: %v", err)
		}
470

Michael Yang's avatar
Michael Yang committed
471
472
473
474
475
476
		for k, v := range sourceParams {
			if _, ok := formattedParams[k]; !ok {
				formattedParams[k] = v
			}
		}

Michael Yang's avatar
Michael Yang committed
477
478
479
480
481
482
		if config.ModelType == "65B" {
			if numGQA, ok := formattedParams["num_gqa"].(int); ok && numGQA == 8 {
				config.ModelType = "70B"
			}
		}

483
484
485
486
487
488
		bts, err := json.Marshal(formattedParams)
		if err != nil {
			return err
		}

		l, err := CreateLayer(bytes.NewReader(bts))
489
490
491
492
493
		if err != nil {
			return fmt.Errorf("failed to create layer: %v", err)
		}
		l.MediaType = "application/vnd.ollama.image.params"
		layers = append(layers, l)
494
495
	}

496
497
498
499
500
501
502
503
	digests, err := getLayerDigests(layers)
	if err != nil {
		return err
	}

	var manifestLayers []*Layer
	for _, l := range layers {
		manifestLayers = append(manifestLayers, &l.Layer)
504
		delete(deleteMap, l.Layer.Digest)
505
506
507
	}

	// Create a layer for the config object
508
	fn(api.ProgressResponse{Status: "creating config layer"})
509
	cfg, err := createConfigLayer(config, digests)
510
511
512
513
	if err != nil {
		return err
	}
	layers = append(layers, cfg)
514
	delete(deleteMap, cfg.Layer.Digest)
515

Michael Yang's avatar
Michael Yang committed
516
	if err := SaveLayers(layers, fn, false); err != nil {
517
518
519
520
		return err
	}

	// Create the manifest
521
	fn(api.ProgressResponse{Status: "writing manifest"})
522
523
524
525
526
	err = CreateManifest(name, cfg, manifestLayers)
	if err != nil {
		return err
	}

527
528
529
530
531
532
533
534
	if noprune == "" {
		fn(api.ProgressResponse{Status: "removing any unused layers"})
		err = deleteUnusedLayers(nil, deleteMap, false)
		if err != nil {
			return err
		}
	}

535
	fn(api.ProgressResponse{Status: "success"})
536
537
538
	return nil
}

Michael Yang's avatar
Michael Yang committed
539
func removeLayerFromLayers(layers []*LayerReader, mediaType string) []*LayerReader {
Michael Yang's avatar
Michael Yang committed
540
541
542
	return slices.DeleteFunc(layers, func(layer *LayerReader) bool {
		return layer.MediaType == mediaType
	})
543
544
}

545
func SaveLayers(layers []*LayerReader, fn func(resp api.ProgressResponse), force bool) error {
546
547
	// Write each of the layers to disk
	for _, layer := range layers {
Patrick Devine's avatar
Patrick Devine committed
548
		fp, err := GetBlobsPath(layer.Digest)
549
550
551
		if err != nil {
			return err
		}
552
553

		_, err = os.Stat(fp)
554
		if os.IsNotExist(err) || force {
555
556
			fn(api.ProgressResponse{Status: fmt.Sprintf("writing layer %s", layer.Digest)})

557
558
559
560
561
562
563
			out, err := os.Create(fp)
			if err != nil {
				log.Printf("couldn't create %s", fp)
				return err
			}
			defer out.Close()

Michael Yang's avatar
Michael Yang committed
564
			if _, err = io.Copy(out, layer.Reader); err != nil {
565
566
				return err
			}
Michael Yang's avatar
Michael Yang committed
567

568
		} else {
569
			fn(api.ProgressResponse{Status: fmt.Sprintf("using already created layer %s", layer.Digest)})
570
571
572
573
574
575
		}
	}

	return nil
}

Michael Yang's avatar
Michael Yang committed
576
func CreateManifest(name string, cfg *LayerReader, layers []*Layer) error {
577
	mp := ParseModelPath(name)
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
	manifest := ManifestV2{
		SchemaVersion: 2,
		MediaType:     "application/vnd.docker.distribution.manifest.v2+json",
		Config: Layer{
			MediaType: cfg.MediaType,
			Size:      cfg.Size,
			Digest:    cfg.Digest,
		},
		Layers: layers,
	}

	manifestJSON, err := json.Marshal(manifest)
	if err != nil {
		return err
	}

594
	fp, err := mp.GetManifestPath()
595
596
597
	if err != nil {
		return err
	}
598
599
600
	if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
		return err
	}
601
	return os.WriteFile(fp, manifestJSON, 0o644)
602
603
}

Michael Yang's avatar
Michael Yang committed
604
func GetLayerWithBufferFromLayer(layer *Layer) (*LayerReader, error) {
Patrick Devine's avatar
Patrick Devine committed
605
	fp, err := GetBlobsPath(layer.Digest)
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
	if err != nil {
		return nil, err
	}

	file, err := os.Open(fp)
	if err != nil {
		return nil, fmt.Errorf("could not open blob: %w", err)
	}
	defer file.Close()

	newLayer, err := CreateLayer(file)
	if err != nil {
		return nil, err
	}
	newLayer.MediaType = layer.MediaType
	return newLayer, nil
}

624
625
// formatParams converts specified parameter options to their correct types
func formatParams(params map[string][]string) (map[string]interface{}, error) {
626
627
628
	opts := api.Options{}
	valueOpts := reflect.ValueOf(&opts).Elem() // names of the fields in the options struct
	typeOpts := reflect.TypeOf(opts)           // types of the fields in the options struct
Michael Yang's avatar
Michael Yang committed
629

630
	// build map of json struct tags to their types
Michael Yang's avatar
Michael Yang committed
631
632
633
634
635
636
637
638
	jsonOpts := make(map[string]reflect.StructField)
	for _, field := range reflect.VisibleFields(typeOpts) {
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
		if jsonTag != "" {
			jsonOpts[jsonTag] = field
		}
	}

639
	out := make(map[string]interface{})
Michael Yang's avatar
Michael Yang committed
640
	// iterate params and set values based on json struct tags
641
	for key, vals := range params {
Michael Yang's avatar
Michael Yang committed
642
643
644
645
646
		if opt, ok := jsonOpts[key]; ok {
			field := valueOpts.FieldByName(opt.Name)
			if field.IsValid() && field.CanSet() {
				switch field.Kind() {
				case reflect.Float32:
647
					floatVal, err := strconv.ParseFloat(vals[0], 32)
Michael Yang's avatar
Michael Yang committed
648
					if err != nil {
649
						return nil, fmt.Errorf("invalid float value %s", vals)
Michael Yang's avatar
Michael Yang committed
650
651
					}

Michael Yang's avatar
Michael Yang committed
652
					out[key] = float32(floatVal)
Michael Yang's avatar
Michael Yang committed
653
				case reflect.Int:
Michael Yang's avatar
Michael Yang committed
654
					intVal, err := strconv.ParseInt(vals[0], 10, 64)
Michael Yang's avatar
Michael Yang committed
655
					if err != nil {
656
						return nil, fmt.Errorf("invalid int value %s", vals)
Michael Yang's avatar
Michael Yang committed
657
658
					}

659
					out[key] = intVal
Michael Yang's avatar
Michael Yang committed
660
				case reflect.Bool:
661
					boolVal, err := strconv.ParseBool(vals[0])
Michael Yang's avatar
Michael Yang committed
662
					if err != nil {
663
						return nil, fmt.Errorf("invalid bool value %s", vals)
Michael Yang's avatar
Michael Yang committed
664
665
					}

666
					out[key] = boolVal
Michael Yang's avatar
Michael Yang committed
667
				case reflect.String:
668
					out[key] = vals[0]
669
				case reflect.Slice:
670
671
					// TODO: only string slices are supported right now
					out[key] = vals
Michael Yang's avatar
Michael Yang committed
672
673
674
675
676
677
678
				default:
					return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
				}
			}
		}
	}

679
	return out, nil
680
681
}

Michael Yang's avatar
Michael Yang committed
682
func getLayerDigests(layers []*LayerReader) ([]string, error) {
683
684
685
686
687
688
689
690
691
692
693
	var digests []string
	for _, l := range layers {
		if l.Digest == "" {
			return nil, fmt.Errorf("layer is missing a digest")
		}
		digests = append(digests, l.Digest)
	}
	return digests, nil
}

// CreateLayer creates a Layer object from a given file
Michael Yang's avatar
Michael Yang committed
694
695
func CreateLayer(f io.ReadSeeker) (*LayerReader, error) {
	digest, size := GetSHA256Digest(f)
696
	f.Seek(0, io.SeekStart)
697

Michael Yang's avatar
Michael Yang committed
698
	layer := &LayerReader{
699
700
701
702
703
		Layer: Layer{
			MediaType: "application/vnd.docker.image.rootfs.diff.tar",
			Digest:    digest,
			Size:      size,
		},
Michael Yang's avatar
Michael Yang committed
704
		Reader: f,
705
706
707
708
709
	}

	return layer, nil
}

Patrick Devine's avatar
Patrick Devine committed
710
func CopyModel(src, dest string) error {
711
	srcModelPath := ParseModelPath(src)
712
	srcPath, err := srcModelPath.GetManifestPath()
713
714
715
716
	if err != nil {
		return err
	}

717
	destModelPath := ParseModelPath(dest)
718
	destPath, err := destModelPath.GetManifestPath()
Patrick Devine's avatar
Patrick Devine committed
719
720
721
	if err != nil {
		return err
	}
722
723
724
	if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
		return err
	}
Patrick Devine's avatar
Patrick Devine committed
725
726

	// copy the file
Michael Yang's avatar
Michael Yang committed
727
	input, err := os.ReadFile(srcPath)
Patrick Devine's avatar
Patrick Devine committed
728
729
730
731
732
	if err != nil {
		fmt.Println("Error reading file:", err)
		return err
	}

Michael Yang's avatar
Michael Yang committed
733
	err = os.WriteFile(destPath, input, 0o644)
Patrick Devine's avatar
Patrick Devine committed
734
735
736
737
738
739
740
741
	if err != nil {
		fmt.Println("Error reading file:", err)
		return err
	}

	return nil
}

742
func deleteUnusedLayers(skipModelPath *ModelPath, deleteMap map[string]bool, dryRun bool) error {
743
744
745
746
	fp, err := GetManifestPath()
	if err != nil {
		return err
	}
Michael Yang's avatar
Michael Yang committed
747
748
749
750

	walkFunc := func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			return nil
751
752
		}

Michael Yang's avatar
Michael Yang committed
753
754
755
756
		dir, file := filepath.Split(path)
		dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
		tag := strings.Join([]string{dir, file}, ":")
		fmp := ParseModelPath(tag)
757

Michael Yang's avatar
Michael Yang committed
758
		// skip the manifest we're trying to delete
759
		if skipModelPath != nil && skipModelPath.GetFullTagname() == fmp.GetFullTagname() {
Michael Yang's avatar
Michael Yang committed
760
			return nil
761
		}
Michael Yang's avatar
Michael Yang committed
762
763
764
765
766
767
768
769
770
771
772
773

		// save (i.e. delete from the deleteMap) any files used in other manifests
		manifest, _, err := GetManifest(fmp)
		if err != nil {
			return nil
		}

		for _, layer := range manifest.Layers {
			delete(deleteMap, layer.Digest)
		}

		delete(deleteMap, manifest.Config.Digest)
774
		return nil
Michael Yang's avatar
Michael Yang committed
775
776
777
	}

	if err := filepath.Walk(fp, walkFunc); err != nil {
Michael Yang's avatar
Michael Yang committed
778
779
		return err
	}
780
781
782
783

	// only delete the files which are still in the deleteMap
	for k, v := range deleteMap {
		if v {
784
			fp, err := GetBlobsPath(k)
785
			if err != nil {
786
787
788
				log.Printf("couldn't get file path for '%s': %v", k, err)
				continue
			}
789
790
791
792
793
794
795
			if !dryRun {
				if err := os.Remove(fp); err != nil {
					log.Printf("couldn't remove file '%s': %v", fp, err)
					continue
				}
			} else {
				log.Printf("wanted to remove: %s", fp)
796
797
798
799
			}
		}
	}

800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
	return nil
}

func PruneLayers() error {
	deleteMap := make(map[string]bool)
	p, err := GetBlobsPath("")
	if err != nil {
		return err
	}

	blobs, err := os.ReadDir(p)
	if err != nil {
		log.Printf("couldn't read dir '%s': %v", p, err)
		return err
	}

	for _, blob := range blobs {
		name := blob.Name()
		if runtime.GOOS == "windows" {
			name = strings.ReplaceAll(name, "-", ":")
		}
		deleteMap[name] = true
	}

	log.Printf("total blobs: %d", len(deleteMap))

	err = deleteUnusedLayers(nil, deleteMap, false)
	if err != nil {
		return err
	}

	log.Printf("total unused blobs removed: %d", len(deleteMap))

	return nil
}

Michael Yang's avatar
Michael Yang committed
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
func PruneDirectory(path string) error {
	info, err := os.Lstat(path)
	if err != nil {
		return err
	}

	if info.IsDir() && info.Mode()&os.ModeSymlink == 0 {
		entries, err := os.ReadDir(path)
		if err != nil {
			return err
		}

		for _, entry := range entries {
			if err := PruneDirectory(filepath.Join(path, entry.Name())); err != nil {
				return err
			}
		}

		entries, err = os.ReadDir(path)
		if err != nil {
			return err
		}

		if len(entries) > 0 {
			return nil
		}

		return os.Remove(path)
	}

	return nil
}

869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
func DeleteModel(name string) error {
	mp := ParseModelPath(name)
	manifest, _, err := GetManifest(mp)
	if err != nil {
		return err
	}

	deleteMap := make(map[string]bool)
	for _, layer := range manifest.Layers {
		deleteMap[layer.Digest] = true
	}
	deleteMap[manifest.Config.Digest] = true

	err = deleteUnusedLayers(&mp, deleteMap, false)
	if err != nil {
		return err
	}

887
	fp, err := mp.GetManifestPath()
888
889
890
891
892
893
894
895
896
897
898
899
	if err != nil {
		return err
	}
	err = os.Remove(fp)
	if err != nil {
		log.Printf("couldn't remove manifest file '%s': %v", fp, err)
		return err
	}

	return nil
}

Patrick Devine's avatar
Patrick Devine committed
900
func ShowModelfile(model *Model) (string, error) {
Michael Yang's avatar
Michael Yang committed
901
	var mt struct {
Patrick Devine's avatar
Patrick Devine committed
902
		*Model
Michael Yang's avatar
Michael Yang committed
903
		From       string
Michael Yang's avatar
Michael Yang committed
904
		Parameters map[string][]any
Patrick Devine's avatar
Patrick Devine committed
905
906
	}

Michael Yang's avatar
Michael Yang committed
907
	mt.Parameters = make(map[string][]any)
Patrick Devine's avatar
Patrick Devine committed
908
	for k, v := range model.Options {
Michael Yang's avatar
Michael Yang committed
909
910
911
		if s, ok := v.([]any); ok {
			mt.Parameters[k] = s
			continue
Patrick Devine's avatar
Patrick Devine committed
912
913
		}

Michael Yang's avatar
Michael Yang committed
914
		mt.Parameters[k] = []any{v}
Patrick Devine's avatar
Patrick Devine committed
915
916
	}

Michael Yang's avatar
Michael Yang committed
917
918
	mt.Model = model
	mt.From = model.ModelPath
Patrick Devine's avatar
Patrick Devine committed
919

Michael Yang's avatar
Michael Yang committed
920
921
	if model.OriginalModel != "" {
		mt.From = model.OriginalModel
Patrick Devine's avatar
Patrick Devine committed
922
923
924
925
926
927
928
929
	}

	modelFile := `# Modelfile generated by "ollama show"
# To build a new Modelfile based on this one, replace the FROM line with:
# FROM {{ .ShortName }}

FROM {{ .From }}
TEMPLATE """{{ .Template }}"""
930
931

{{- if .System }}
Patrick Devine's avatar
Patrick Devine committed
932
SYSTEM """{{ .System }}"""
933
{{- end }}
934
935
936
937

{{- range $adapter := .AdapterPaths }}
ADAPTER {{ $adapter }}
{{- end }}
Michael Yang's avatar
Michael Yang committed
938

Michael Yang's avatar
Michael Yang committed
939
940
941
942
{{- range $k, $v := .Parameters }}
{{- range $parameter := $v }}
PARAMETER {{ $k }} {{ printf "%#v" $parameter }}
{{- end }}
Michael Yang's avatar
Michael Yang committed
943
{{- end }}`
Patrick Devine's avatar
Patrick Devine committed
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960

	tmpl, err := template.New("").Parse(modelFile)
	if err != nil {
		log.Printf("error parsing template: %q", err)
		return "", err
	}

	var buf bytes.Buffer

	if err = tmpl.Execute(&buf, mt); err != nil {
		log.Printf("error executing template: %q", err)
		return "", err
	}

	return buf.String(), nil
}

961
func PushModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
962
	mp := ParseModelPath(name)
963
964
	fn(api.ProgressResponse{Status: "retrieving manifest"})

965
966
967
968
	if mp.ProtocolScheme == "http" && !regOpts.Insecure {
		return fmt.Errorf("insecure protocol http")
	}

Patrick Devine's avatar
Patrick Devine committed
969
	manifest, _, err := GetManifest(mp)
970
	if err != nil {
971
		fn(api.ProgressResponse{Status: "couldn't retrieve manifest"})
972
973
974
975
		return err
	}

	var layers []*Layer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
976
	layers = append(layers, manifest.Layers...)
977
978
979
	layers = append(layers, &manifest.Config)

	for _, layer := range layers {
Michael Yang's avatar
Michael Yang committed
980
		if err := uploadBlob(ctx, mp, layer, regOpts, fn); err != nil {
981
982
983
			log.Printf("error uploading blob: %v", err)
			return err
		}
984
985
	}

986
	fn(api.ProgressResponse{Status: "pushing manifest"})
Michael Yang's avatar
Michael Yang committed
987
988
	requestURL := mp.BaseURL()
	requestURL = requestURL.JoinPath("v2", mp.GetNamespaceRepository(), "manifests", mp.Tag)
989
990
991
992
993
994

	manifestJSON, err := json.Marshal(manifest)
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
995
996
	headers := make(http.Header)
	headers.Set("Content-Type", "application/vnd.docker.distribution.manifest.v2+json")
Michael Yang's avatar
Michael Yang committed
997
	resp, err := makeRequestWithRetry(ctx, http.MethodPut, requestURL, headers, bytes.NewReader(manifestJSON), regOpts)
998
999
1000
1001
1002
	if err != nil {
		return err
	}
	defer resp.Body.Close()

1003
	fn(api.ProgressResponse{Status: "success"})
1004
1005
1006
1007

	return nil
}

1008
func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn func(api.ProgressResponse)) error {
1009
1010
	mp := ParseModelPath(name)

1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
	var manifest *ManifestV2
	var err error
	var noprune string

	// build deleteMap to prune unused layers
	deleteMap := make(map[string]bool)

	if noprune = os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		manifest, _, err = GetManifest(mp)
		if err != nil && !errors.Is(err, os.ErrNotExist) {
			return err
		}

		if manifest != nil {
			for _, l := range manifest.Layers {
				deleteMap[l.Digest] = true
			}
			deleteMap[manifest.Config.Digest] = true
		}
	}

1032
1033
	if mp.ProtocolScheme == "http" && !regOpts.Insecure {
		return fmt.Errorf("insecure protocol http")
1034
	}
1035

1036
	fn(api.ProgressResponse{Status: "pulling manifest"})
1037

1038
	manifest, err = pullModelManifest(ctx, mp, regOpts)
1039
	if err != nil {
1040
		return fmt.Errorf("pull model manifest: %s", err)
1041
1042
1043
	}

	var layers []*Layer
Bruce MacDonald's avatar
Bruce MacDonald committed
1044
	layers = append(layers, manifest.Layers...)
1045
1046
1047
	layers = append(layers, &manifest.Config)

	for _, layer := range layers {
1048
1049
1050
1051
1052
1053
1054
1055
		if err := downloadBlob(
			ctx,
			downloadOpts{
				mp:      mp,
				digest:  layer.Digest,
				regOpts: regOpts,
				fn:      fn,
			}); err != nil {
1056
1057
			return err
		}
1058
		delete(deleteMap, layer.Digest)
1059
	}
1060
	delete(deleteMap, manifest.Config.Digest)
1061

Michael Yang's avatar
Michael Yang committed
1062
1063
1064
	fn(api.ProgressResponse{Status: "verifying sha256 digest"})
	for _, layer := range layers {
		if err := verifyBlob(layer.Digest); err != nil {
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
			if errors.Is(err, errDigestMismatch) {
				// something went wrong, delete the blob
				fp, err := GetBlobsPath(layer.Digest)
				if err != nil {
					return err
				}
				if err := os.Remove(fp); err != nil {
					// log this, but return the original error
					log.Printf("couldn't remove file with digest mismatch '%s': %v", fp, err)
				}
			}
Michael Yang's avatar
Michael Yang committed
1076
1077
1078
1079
			return err
		}
	}

1080
	fn(api.ProgressResponse{Status: "writing manifest"})
1081

1082
	manifestJSON, err := json.Marshal(manifest)
1083
1084
1085
1086
	if err != nil {
		return err
	}

1087
	fp, err := mp.GetManifestPath()
1088
1089
1090
	if err != nil {
		return err
	}
1091
1092
1093
	if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
		return err
	}
1094

Bruce MacDonald's avatar
Bruce MacDonald committed
1095
	err = os.WriteFile(fp, manifestJSON, 0o644)
1096
1097
1098
1099
1100
	if err != nil {
		log.Printf("couldn't write to %s", fp)
		return err
	}

1101
1102
1103
1104
1105
1106
1107
1108
	if noprune == "" {
		fn(api.ProgressResponse{Status: "removing any unused layers"})
		err = deleteUnusedLayers(nil, deleteMap, false)
		if err != nil {
			return err
		}
	}

1109
	fn(api.ProgressResponse{Status: "success"})
1110
1111
1112
1113

	return nil
}

1114
func pullModelManifest(ctx context.Context, mp ModelPath, regOpts *RegistryOptions) (*ManifestV2, error) {
Michael Yang's avatar
Michael Yang committed
1115
	requestURL := mp.BaseURL().JoinPath("v2", mp.GetNamespaceRepository(), "manifests", mp.Tag)
1116

Michael Yang's avatar
Michael Yang committed
1117
1118
	headers := make(http.Header)
	headers.Set("Accept", "application/vnd.docker.distribution.manifest.v2+json")
Michael Yang's avatar
Michael Yang committed
1119
	resp, err := makeRequestWithRetry(ctx, http.MethodGet, requestURL, headers, nil, regOpts)
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	var m *ManifestV2
	if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
		return nil, err
	}

	return m, err
}

1133
1134
1135
1136
func createConfigLayer(config ConfigV2, layers []string) (*LayerReader, error) {
	config.RootFS = RootFS{
		Type:    "layers",
		DiffIDs: layers,
1137
1138
1139
1140
1141
1142
1143
	}

	configJSON, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

1144
	digest, size := GetSHA256Digest(bytes.NewBuffer(configJSON))
1145

Michael Yang's avatar
Michael Yang committed
1146
	layer := &LayerReader{
1147
1148
1149
1150
1151
		Layer: Layer{
			MediaType: "application/vnd.docker.container.image.v1+json",
			Digest:    digest,
			Size:      size,
		},
1152
		Reader: bytes.NewBuffer(configJSON),
1153
1154
1155
1156
1157
	}
	return layer, nil
}

// GetSHA256Digest returns the SHA256 hash of a given buffer and returns it, and the size of buffer
Michael Yang's avatar
Michael Yang committed
1158
func GetSHA256Digest(r io.Reader) (string, int64) {
Michael Yang's avatar
Michael Yang committed
1159
1160
1161
1162
1163
1164
	h := sha256.New()
	n, err := io.Copy(h, r)
	if err != nil {
		log.Fatal(err)
	}

Michael Yang's avatar
Michael Yang committed
1165
	return fmt.Sprintf("sha256:%x", h.Sum(nil)), n
1166
1167
}

Michael Yang's avatar
Michael Yang committed
1168
func makeRequestWithRetry(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.ReadSeeker, regOpts *RegistryOptions) (*http.Response, error) {
Michael Yang's avatar
Michael Yang committed
1169
	for try := 0; try < maxRetries; try++ {
Michael Yang's avatar
Michael Yang committed
1170
		resp, err := makeRequest(ctx, method, requestURL, headers, body, regOpts)
Michael Yang's avatar
Michael Yang committed
1171
1172
1173
1174
1175
		if err != nil {
			log.Printf("couldn't start upload: %v", err)
			return nil, err
		}

Michael Yang's avatar
Michael Yang committed
1176
1177
		switch {
		case resp.StatusCode == http.StatusUnauthorized:
Michael Yang's avatar
Michael Yang committed
1178
1179
			auth := resp.Header.Get("www-authenticate")
			authRedir := ParseAuthRedirectString(auth)
Michael Yang's avatar
Michael Yang committed
1180
			token, err := getAuthToken(ctx, authRedir)
Michael Yang's avatar
Michael Yang committed
1181
1182
1183
1184
1185
1186
			if err != nil {
				return nil, err
			}

			regOpts.Token = token
			if body != nil {
Michael Yang's avatar
Michael Yang committed
1187
				body.Seek(0, io.SeekStart)
Michael Yang's avatar
Michael Yang committed
1188
1189
1190
			}

			continue
Michael Yang's avatar
Michael Yang committed
1191
1192
		case resp.StatusCode == http.StatusNotFound:
			return nil, os.ErrNotExist
Michael Yang's avatar
Michael Yang committed
1193
		case resp.StatusCode >= http.StatusBadRequest:
Michael Yang's avatar
Michael Yang committed
1194
1195
1196
1197
1198
1199
			body, err := io.ReadAll(resp.Body)
			if err != nil {
				return nil, fmt.Errorf("%d: %s", resp.StatusCode, err)
			}

			return nil, fmt.Errorf("%d: %s", resp.StatusCode, body)
Michael Yang's avatar
Michael Yang committed
1200
1201
		default:
			return resp, nil
Michael Yang's avatar
Michael Yang committed
1202
1203
1204
		}
	}

Michael Yang's avatar
Michael Yang committed
1205
	return nil, errMaxRetriesExceeded
Michael Yang's avatar
Michael Yang committed
1206
1207
}

Michael Yang's avatar
Michael Yang committed
1208
func makeRequest(ctx context.Context, method string, requestURL *url.URL, headers http.Header, body io.Reader, regOpts *RegistryOptions) (*http.Response, error) {
Michael Yang's avatar
Michael Yang committed
1209
	if requestURL.Scheme != "http" && regOpts != nil && regOpts.Insecure {
Michael Yang's avatar
Michael Yang committed
1210
		requestURL.Scheme = "http"
1211
1212
	}

Michael Yang's avatar
Michael Yang committed
1213
	req, err := http.NewRequestWithContext(ctx, method, requestURL.String(), body)
1214
1215
1216
1217
	if err != nil {
		return nil, err
	}

Michael Yang's avatar
Michael Yang committed
1218
1219
1220
1221
	if headers != nil {
		req.Header = headers
	}

Michael Yang's avatar
Michael Yang committed
1222
1223
1224
1225
1226
1227
	if regOpts != nil {
		if regOpts.Token != "" {
			req.Header.Set("Authorization", "Bearer "+regOpts.Token)
		} else if regOpts.Username != "" && regOpts.Password != "" {
			req.SetBasicAuth(regOpts.Username, regOpts.Password)
		}
1228
1229
	}

Michael Yang's avatar
Michael Yang committed
1230
	req.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
1231

Michael Yang's avatar
Michael Yang committed
1232
1233
1234
1235
1236
1237
1238
1239
1240
	if s := req.Header.Get("Content-Length"); s != "" {
		contentLength, err := strconv.ParseInt(s, 10, 64)
		if err != nil {
			return nil, err
		}

		req.ContentLength = contentLength
	}

Michael Yang's avatar
Michael Yang committed
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
	proxyURL, err := http.ProxyFromEnvironment(req)
	if err != nil {
		return nil, err
	}

	client := http.Client{
		Transport: &http.Transport{
			Proxy: http.ProxyURL(proxyURL),
		},
	}

	resp, err := client.Do(req)
1253
1254
1255
1256
1257
1258
	if err != nil {
		return nil, err
	}

	return resp, nil
}
Michael Yang's avatar
Michael Yang committed
1259

Patrick Devine's avatar
Patrick Devine committed
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
func getValue(header, key string) string {
	startIdx := strings.Index(header, key+"=")
	if startIdx == -1 {
		return ""
	}

	// Move the index to the starting quote after the key.
	startIdx += len(key) + 2
	endIdx := startIdx

	for endIdx < len(header) {
		if header[endIdx] == '"' {
			if endIdx+1 < len(header) && header[endIdx+1] != ',' { // If the next character isn't a comma, continue
				endIdx++
				continue
			}
			break
		}
		endIdx++
	}
	return header[startIdx:endIdx]
}

func ParseAuthRedirectString(authStr string) AuthRedirect {
	authStr = strings.TrimPrefix(authStr, "Bearer ")

	return AuthRedirect{
		Realm:   getValue(authStr, "realm"),
		Service: getValue(authStr, "service"),
		Scope:   getValue(authStr, "scope"),
	}
}

1293
1294
var errDigestMismatch = fmt.Errorf("digest mismatch, file must be downloaded again")

Michael Yang's avatar
Michael Yang committed
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
func verifyBlob(digest string) error {
	fp, err := GetBlobsPath(digest)
	if err != nil {
		return err
	}

	f, err := os.Open(fp)
	if err != nil {
		return err
	}
	defer f.Close()

	fileDigest, _ := GetSHA256Digest(f)
	if digest != fileDigest {
1309
		return fmt.Errorf("%w: want %s, got %s", errDigestMismatch, digest, fileDigest)
Michael Yang's avatar
Michael Yang committed
1310
1311
1312
1313
	}

	return nil
}