model.go 8.38 KB
Newer Older
Michael Yang's avatar
Michael Yang committed
1
2
3
4
5
6
package server

import (
	"archive/zip"
	"bytes"
	"context"
Michael Yang's avatar
tools  
Michael Yang committed
7
	"encoding/json"
Michael Yang's avatar
Michael Yang committed
8
9
10
	"errors"
	"fmt"
	"io"
11
	"log/slog"
Michael Yang's avatar
Michael Yang committed
12
13
14
	"net/http"
	"os"
	"path/filepath"
Michael Yang's avatar
tools  
Michael Yang committed
15
16
17
	"slices"
	"strings"
	"text/template/parse"
Michael Yang's avatar
Michael Yang committed
18
19
20
21

	"github.com/ollama/ollama/api"
	"github.com/ollama/ollama/convert"
	"github.com/ollama/ollama/llm"
Michael Yang's avatar
Michael Yang committed
22
	"github.com/ollama/ollama/template"
Michael Yang's avatar
Michael Yang committed
23
24
25
	"github.com/ollama/ollama/types/model"
)

Michael Yang's avatar
Michael Yang committed
26
var intermediateBlobs map[string]string = make(map[string]string)
27

28
type layerGGML struct {
Michael Yang's avatar
Michael Yang committed
29
30
31
32
	*Layer
	*llm.GGML
}

33
func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
34
	m, err := ParseNamedManifest(name)
Michael Yang's avatar
Michael Yang committed
35
36
	switch {
	case errors.Is(err, os.ErrNotExist):
Michael Yang's avatar
Michael Yang committed
37
		if err := PullModel(ctx, name.String(), &registryOptions{}, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
38
39
40
			return nil, err
		}

41
		m, err = ParseNamedManifest(name)
Michael Yang's avatar
Michael Yang committed
42
43
44
		if err != nil {
			return nil, err
		}
Michael Yang's avatar
Michael Yang committed
45
46
47
48
	case err != nil:
		return nil, err
	}

49
50
	for _, layer := range m.Layers {
		layer, err := NewLayerFromLayer(layer.Digest, layer.MediaType, name.DisplayShortest())
Michael Yang's avatar
Michael Yang committed
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
		if err != nil {
			return nil, err
		}

		switch layer.MediaType {
		case "application/vnd.ollama.image.model",
			"application/vnd.ollama.image.projector",
			"application/vnd.ollama.image.adapter":
			blobpath, err := GetBlobsPath(layer.Digest)
			if err != nil {
				return nil, err
			}

			blob, err := os.Open(blobpath)
			if err != nil {
				return nil, err
			}
			defer blob.Close()

70
			ggml, _, err := llm.DecodeGGML(blob, 0)
Michael Yang's avatar
Michael Yang committed
71
72
73
			if err != nil {
				return nil, err
			}
Michael Yang's avatar
Michael Yang committed
74

75
			layers = append(layers, &layerGGML{layer, ggml})
Michael Yang's avatar
Michael Yang committed
76
		default:
77
			layers = append(layers, &layerGGML{layer, nil})
Michael Yang's avatar
Michael Yang committed
78
79
80
81
82
83
		}
	}

	return layers, nil
}

84
func extractFromZipFile(p string, file *os.File, fn func(api.ProgressResponse)) error {
Michael Yang's avatar
Michael Yang committed
85
86
	stat, err := file.Stat()
	if err != nil {
87
		return err
Michael Yang's avatar
Michael Yang committed
88
89
90
91
	}

	r, err := zip.NewReader(file, stat.Size())
	if err != nil {
92
		return err
Michael Yang's avatar
Michael Yang committed
93
94
95
96
	}

	fn(api.ProgressResponse{Status: "unpacking model metadata"})
	for _, f := range r.File {
Michael Yang's avatar
Michael Yang committed
97
98
		if !filepath.IsLocal(f.Name) {
			return fmt.Errorf("%w: %s", zip.ErrInsecurePath, f.Name)
99
100
		}

Michael Yang's avatar
Michael Yang committed
101
		n := filepath.Join(p, f.Name)
102
103
104
105
		if err := os.MkdirAll(filepath.Dir(n), 0o750); err != nil {
			return err
		}

Michael Yang's avatar
Michael Yang committed
106
		// TODO(mxyng): this should not write out all files to disk
107
		outfile, err := os.Create(n)
Michael Yang's avatar
Michael Yang committed
108
		if err != nil {
109
			return err
Michael Yang's avatar
Michael Yang committed
110
		}
Michael Yang's avatar
Michael Yang committed
111
		defer outfile.Close()
Michael Yang's avatar
Michael Yang committed
112
113
114

		infile, err := f.Open()
		if err != nil {
115
			return err
Michael Yang's avatar
Michael Yang committed
116
		}
Michael Yang's avatar
Michael Yang committed
117
		defer infile.Close()
Michael Yang's avatar
Michael Yang committed
118
119

		if _, err = io.Copy(outfile, infile); err != nil {
120
			return err
Michael Yang's avatar
Michael Yang committed
121
122
123
		}

		if err := outfile.Close(); err != nil {
124
			return err
Michael Yang's avatar
Michael Yang committed
125
126
127
		}

		if err := infile.Close(); err != nil {
128
			return err
Michael Yang's avatar
Michael Yang committed
129
130
131
		}
	}

132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
	return nil
}

func parseFromZipFile(_ context.Context, file *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
	tempDir, err := os.MkdirTemp(filepath.Dir(file.Name()), "")
	if err != nil {
		return nil, err
	}
	defer os.RemoveAll(tempDir)

	if err := extractFromZipFile(tempDir, file, fn); err != nil {
		return nil, err
	}

	mf, err := convert.GetModelFormat(tempDir)
Michael Yang's avatar
Michael Yang committed
147
148
149
150
	if err != nil {
		return nil, err
	}

151
	params, err := mf.GetParams(tempDir)
Michael Yang's avatar
Michael Yang committed
152
153
154
155
	if err != nil {
		return nil, err
	}

156
	mArch, err := mf.GetModelArch("", tempDir, params)
Michael Yang's avatar
Michael Yang committed
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
	if err != nil {
		return nil, err
	}

	fn(api.ProgressResponse{Status: "processing tensors"})
	if err := mArch.GetTensors(); err != nil {
		return nil, err
	}

	if err := mArch.LoadVocab(); err != nil {
		return nil, err
	}

	fn(api.ProgressResponse{Status: "converting model"})

	// TODO(mxyng): this should write directly into a layer
	// e.g. NewLayer(arch.Reader(), "application/vnd.ollama.image.model")
174
	temp, err := os.CreateTemp(tempDir, "fp16")
Michael Yang's avatar
Michael Yang committed
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
	if err != nil {
		return nil, err
	}
	defer temp.Close()
	defer os.Remove(temp.Name())

	if err = mArch.WriteGGUF(temp); err != nil {
		return nil, err
	}

	if _, err := temp.Seek(0, io.SeekStart); err != nil {
		return nil, err
	}

	layer, err := NewLayer(temp, "application/vnd.ollama.image.model")
	if err != nil {
Michael Yang's avatar
Michael Yang committed
191
		return nil, err
Michael Yang's avatar
Michael Yang committed
192
193
	}

194
	bin, err := layer.Open()
Michael Yang's avatar
Michael Yang committed
195
196
197
198
199
	if err != nil {
		return nil, err
	}
	defer bin.Close()

200
	ggml, _, err := llm.DecodeGGML(bin, 0)
Michael Yang's avatar
Michael Yang committed
201
202
203
204
	if err != nil {
		return nil, err
	}

205
	layers = append(layers, &layerGGML{layer, ggml})
206

Michael Yang's avatar
Michael Yang committed
207
	intermediateBlobs[digest] = layer.Digest
208
	return detectChatTemplate(layers)
Michael Yang's avatar
Michael Yang committed
209
210
}

211
func parseFromFile(ctx context.Context, file *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
Michael Yang's avatar
Michael Yang committed
212
213
214
215
216
217
218
219
220
221
	sr := io.NewSectionReader(file, 0, 512)
	contentType, err := detectContentType(sr)
	if err != nil {
		return nil, err
	}

	switch contentType {
	case "gguf", "ggla":
		// noop
	case "application/zip":
222
		return parseFromZipFile(ctx, file, digest, fn)
Michael Yang's avatar
Michael Yang committed
223
224
225
226
227
228
229
230
231
232
233
	default:
		return nil, fmt.Errorf("unsupported content type: %s", contentType)
	}

	stat, err := file.Stat()
	if err != nil {
		return nil, err
	}

	var offset int64
	for offset < stat.Size() {
234
		ggml, n, err := llm.DecodeGGML(file, 0)
Michael Yang's avatar
Michael Yang committed
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
		if errors.Is(err, io.EOF) {
			break
		} else if err != nil {
			return nil, err
		}

		mediatype := "application/vnd.ollama.image.model"
		if ggml.Name() == "ggla" {
			mediatype = "application/vnd.ollama.image.adapter"
		} else if ggml.KV().Architecture() == "clip" {
			mediatype = "application/vnd.ollama.image.projector"
		}

		layer, err := NewLayer(io.NewSectionReader(file, offset, n), mediatype)
		if err != nil {
			return nil, err
		}

253
		layers = append(layers, &layerGGML{layer, ggml})
Michael Yang's avatar
Michael Yang committed
254
255
256
		offset = n
	}

257
258
259
260
261
262
	return detectChatTemplate(layers)
}

func detectChatTemplate(layers []*layerGGML) ([]*layerGGML, error) {
	for _, layer := range layers {
		if s := layer.GGML.KV().ChatTemplate(); s != "" {
Michael Yang's avatar
Michael Yang committed
263
			if t, err := template.Named(s); err != nil {
264
265
266
267
268
269
270
271
272
273
274
275
276
				slog.Debug("template detection", "error", err)
			} else {
				tmpl, err := NewLayer(t.Reader(), "application/vnd.ollama.image.template")
				if err != nil {
					return nil, err
				}

				tmpl.status = fmt.Sprintf("using autodetected template %s", t.Name)
				layers = append(layers, &layerGGML{tmpl, nil})
			}
		}
	}

Michael Yang's avatar
Michael Yang committed
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
	return layers, nil
}

func detectContentType(r io.Reader) (string, error) {
	var b bytes.Buffer
	if _, err := io.Copy(&b, r); err != nil {
		return "", err
	}

	if contentType := llm.DetectGGMLType(b.Bytes()); contentType != "" {
		return contentType, nil
	}

	if contentType := http.DetectContentType(b.Bytes()); contentType != "application/octet-stream" {
		return contentType, nil
	}

	return "unknown", nil
}
Michael Yang's avatar
tools  
Michael Yang committed
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328

// parseToolCalls attempts to parse a JSON string into a slice of ToolCalls.
// mxyng: this only really works if the input contains tool calls in some JSON format
func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) {
	// create a subtree from the node that ranges over .ToolCalls
	tmpl := m.Template.Subtree(func(n parse.Node) bool {
		if t, ok := n.(*parse.RangeNode); ok {
			return slices.Contains(template.Identifiers(t.Pipe), "ToolCalls")
		}

		return false
	})

	if tmpl == nil {
		return nil, false
	}

	var b bytes.Buffer
	if err := tmpl.Execute(&b, map[string][]map[string]any{
		"ToolCalls": {
			{
				"Function": map[string]any{
					"Name":      "@@name@@",
					"Arguments": "@@arguments@@",
				},
			},
		},
	}); err != nil {
		return nil, false
	}

	var kv map[string]string
	// execute the subtree with placeholders to identify the keys
Michael Yang's avatar
Michael Yang committed
329
330
	// trim any commands that might exist in the template
	if err := json.Unmarshal(bytes.TrimSuffix(b.Bytes(), []byte(",")), &kv); err != nil {
Michael Yang's avatar
tools  
Michael Yang committed
331
332
333
334
335
336
337
338
339
340
341
342
343
344
		return nil, false
	}

	// find the keys that correspond to the name and arguments fields
	var name, arguments string
	for k, v := range kv {
		switch v {
		case "@@name@@":
			name = k
		case "@@arguments@@":
			arguments = k
		}
	}

Michael Yang's avatar
Michael Yang committed
345
346
347
348
349
350
351
352
353
354
355
	var objs []map[string]any
	for offset := 0; offset < len(s); {
		if err := json.NewDecoder(strings.NewReader(s[offset:])).Decode(&objs); errors.Is(err, io.EOF) {
			break
		} else if syntax := &(json.SyntaxError{}); errors.As(err, &syntax) {
			// skip over any syntax errors
			offset += int(syntax.Offset)
		} else if unmarshalType := &(json.UnmarshalTypeError{}); errors.As(err, &unmarshalType) {
			// skip over any unmarshalable types
			offset += int(unmarshalType.Offset)
		} else if err != nil {
Michael Yang's avatar
tools  
Michael Yang committed
356
			return nil, false
Michael Yang's avatar
Michael Yang committed
357
358
359
		} else {
			// break when an object is decoded
			break
Michael Yang's avatar
tools  
Michael Yang committed
360
361
362
363
		}
	}

	var toolCalls []api.ToolCall
Michael Yang's avatar
Michael Yang committed
364
	for _, kv := range objs {
Michael Yang's avatar
Michael Yang committed
365
		var call api.ToolCall
Michael Yang's avatar
tools  
Michael Yang committed
366
367
368
369
370
371
372
373
374
375
376
377
		for k, v := range kv {
			switch k {
			case name:
				call.Function.Name = v.(string)
			case arguments:
				call.Function.Arguments = v.(map[string]any)
			}
		}

		toolCalls = append(toolCalls, call)
	}

Michael Yang's avatar
Michael Yang committed
378
	return toolCalls, len(toolCalls) > 0
Michael Yang's avatar
tools  
Michael Yang committed
379
}