model.go 9.03 KB
Newer Older
Michael Yang's avatar
Michael Yang committed
1
2
3
4
5
6
package server

import (
	"archive/zip"
	"bytes"
	"context"
Michael Yang's avatar
tools  
Michael Yang committed
7
	"encoding/json"
Michael Yang's avatar
Michael Yang committed
8
9
10
	"errors"
	"fmt"
	"io"
11
	"log/slog"
Michael Yang's avatar
Michael Yang committed
12
13
14
	"net/http"
	"os"
	"path/filepath"
Michael Yang's avatar
tools  
Michael Yang committed
15
16
17
	"slices"
	"strings"
	"text/template/parse"
Michael Yang's avatar
Michael Yang committed
18
19
20
21

	"github.com/ollama/ollama/api"
	"github.com/ollama/ollama/convert"
	"github.com/ollama/ollama/llm"
Michael Yang's avatar
Michael Yang committed
22
	"github.com/ollama/ollama/template"
Michael Yang's avatar
Michael Yang committed
23
24
25
	"github.com/ollama/ollama/types/model"
)

Michael Yang's avatar
Michael Yang committed
26
var intermediateBlobs map[string]string = make(map[string]string)
27

28
type layerGGML struct {
Michael Yang's avatar
Michael Yang committed
29
30
31
32
	*Layer
	*llm.GGML
}

33
func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
34
	m, err := ParseNamedManifest(name)
Michael Yang's avatar
Michael Yang committed
35
36
	switch {
	case errors.Is(err, os.ErrNotExist):
Michael Yang's avatar
Michael Yang committed
37
		if err := PullModel(ctx, name.String(), &registryOptions{}, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
38
39
40
			return nil, err
		}

41
		m, err = ParseNamedManifest(name)
Michael Yang's avatar
Michael Yang committed
42
43
44
		if err != nil {
			return nil, err
		}
Michael Yang's avatar
Michael Yang committed
45
46
47
48
	case err != nil:
		return nil, err
	}

49
50
	for _, layer := range m.Layers {
		layer, err := NewLayerFromLayer(layer.Digest, layer.MediaType, name.DisplayShortest())
Michael Yang's avatar
Michael Yang committed
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
		if err != nil {
			return nil, err
		}

		switch layer.MediaType {
		case "application/vnd.ollama.image.model",
			"application/vnd.ollama.image.projector",
			"application/vnd.ollama.image.adapter":
			blobpath, err := GetBlobsPath(layer.Digest)
			if err != nil {
				return nil, err
			}

			blob, err := os.Open(blobpath)
			if err != nil {
				return nil, err
			}
			defer blob.Close()

70
			ggml, _, err := llm.DecodeGGML(blob, 0)
Michael Yang's avatar
Michael Yang committed
71
72
73
			if err != nil {
				return nil, err
			}
Michael Yang's avatar
Michael Yang committed
74

75
			layers = append(layers, &layerGGML{layer, ggml})
Michael Yang's avatar
Michael Yang committed
76
		default:
77
			layers = append(layers, &layerGGML{layer, nil})
Michael Yang's avatar
Michael Yang committed
78
79
80
81
82
83
		}
	}

	return layers, nil
}

84
func extractFromZipFile(p string, file *os.File, fn func(api.ProgressResponse)) error {
Michael Yang's avatar
Michael Yang committed
85
86
	stat, err := file.Stat()
	if err != nil {
87
		return err
Michael Yang's avatar
Michael Yang committed
88
89
90
91
	}

	r, err := zip.NewReader(file, stat.Size())
	if err != nil {
92
		return err
Michael Yang's avatar
Michael Yang committed
93
94
95
96
	}

	fn(api.ProgressResponse{Status: "unpacking model metadata"})
	for _, f := range r.File {
Michael Yang's avatar
Michael Yang committed
97
98
		if !filepath.IsLocal(f.Name) {
			return fmt.Errorf("%w: %s", zip.ErrInsecurePath, f.Name)
99
100
		}

Michael Yang's avatar
Michael Yang committed
101
		n := filepath.Join(p, f.Name)
102
103
104
105
		if err := os.MkdirAll(filepath.Dir(n), 0o750); err != nil {
			return err
		}

Michael Yang's avatar
Michael Yang committed
106
		// TODO(mxyng): this should not write out all files to disk
107
		outfile, err := os.Create(n)
Michael Yang's avatar
Michael Yang committed
108
		if err != nil {
109
			return err
Michael Yang's avatar
Michael Yang committed
110
		}
Michael Yang's avatar
Michael Yang committed
111
		defer outfile.Close()
Michael Yang's avatar
Michael Yang committed
112
113
114

		infile, err := f.Open()
		if err != nil {
115
			return err
Michael Yang's avatar
Michael Yang committed
116
		}
Michael Yang's avatar
Michael Yang committed
117
		defer infile.Close()
Michael Yang's avatar
Michael Yang committed
118
119

		if _, err = io.Copy(outfile, infile); err != nil {
120
			return err
Michael Yang's avatar
Michael Yang committed
121
122
123
		}

		if err := outfile.Close(); err != nil {
124
			return err
Michael Yang's avatar
Michael Yang committed
125
126
127
		}

		if err := infile.Close(); err != nil {
128
			return err
Michael Yang's avatar
Michael Yang committed
129
130
131
		}
	}

132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
	return nil
}

func parseFromZipFile(_ context.Context, file *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
	tempDir, err := os.MkdirTemp(filepath.Dir(file.Name()), "")
	if err != nil {
		return nil, err
	}
	defer os.RemoveAll(tempDir)

	if err := extractFromZipFile(tempDir, file, fn); err != nil {
		return nil, err
	}

	mf, err := convert.GetModelFormat(tempDir)
Michael Yang's avatar
Michael Yang committed
147
148
149
150
	if err != nil {
		return nil, err
	}

151
	params, err := mf.GetParams(tempDir)
Michael Yang's avatar
Michael Yang committed
152
153
154
155
	if err != nil {
		return nil, err
	}

156
	mArch, err := mf.GetModelArch("", tempDir, params)
Michael Yang's avatar
Michael Yang committed
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
	if err != nil {
		return nil, err
	}

	fn(api.ProgressResponse{Status: "processing tensors"})
	if err := mArch.GetTensors(); err != nil {
		return nil, err
	}

	if err := mArch.LoadVocab(); err != nil {
		return nil, err
	}

	fn(api.ProgressResponse{Status: "converting model"})

	// TODO(mxyng): this should write directly into a layer
	// e.g. NewLayer(arch.Reader(), "application/vnd.ollama.image.model")
174
	temp, err := os.CreateTemp(tempDir, "fp16")
Michael Yang's avatar
Michael Yang committed
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
	if err != nil {
		return nil, err
	}
	defer temp.Close()
	defer os.Remove(temp.Name())

	if err = mArch.WriteGGUF(temp); err != nil {
		return nil, err
	}

	if _, err := temp.Seek(0, io.SeekStart); err != nil {
		return nil, err
	}

	layer, err := NewLayer(temp, "application/vnd.ollama.image.model")
	if err != nil {
Michael Yang's avatar
Michael Yang committed
191
		return nil, err
Michael Yang's avatar
Michael Yang committed
192
193
	}

194
	bin, err := layer.Open()
Michael Yang's avatar
Michael Yang committed
195
196
197
198
199
	if err != nil {
		return nil, err
	}
	defer bin.Close()

200
	ggml, _, err := llm.DecodeGGML(bin, 0)
Michael Yang's avatar
Michael Yang committed
201
202
203
204
	if err != nil {
		return nil, err
	}

205
	layers = append(layers, &layerGGML{layer, ggml})
206

Michael Yang's avatar
Michael Yang committed
207
	intermediateBlobs[digest] = layer.Digest
208
	return detectChatTemplate(layers)
Michael Yang's avatar
Michael Yang committed
209
210
}

211
func parseFromFile(ctx context.Context, file *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
Michael Yang's avatar
Michael Yang committed
212
213
214
215
216
217
218
219
220
221
	sr := io.NewSectionReader(file, 0, 512)
	contentType, err := detectContentType(sr)
	if err != nil {
		return nil, err
	}

	switch contentType {
	case "gguf", "ggla":
		// noop
	case "application/zip":
222
		return parseFromZipFile(ctx, file, digest, fn)
Michael Yang's avatar
Michael Yang committed
223
224
225
226
227
228
229
230
231
232
233
	default:
		return nil, fmt.Errorf("unsupported content type: %s", contentType)
	}

	stat, err := file.Stat()
	if err != nil {
		return nil, err
	}

	var offset int64
	for offset < stat.Size() {
234
		ggml, n, err := llm.DecodeGGML(file, 0)
Michael Yang's avatar
Michael Yang committed
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
		if errors.Is(err, io.EOF) {
			break
		} else if err != nil {
			return nil, err
		}

		mediatype := "application/vnd.ollama.image.model"
		if ggml.Name() == "ggla" {
			mediatype = "application/vnd.ollama.image.adapter"
		} else if ggml.KV().Architecture() == "clip" {
			mediatype = "application/vnd.ollama.image.projector"
		}

		layer, err := NewLayer(io.NewSectionReader(file, offset, n), mediatype)
		if err != nil {
			return nil, err
		}

253
		layers = append(layers, &layerGGML{layer, ggml})
Michael Yang's avatar
Michael Yang committed
254
255
256
		offset = n
	}

257
258
259
260
261
262
	return detectChatTemplate(layers)
}

func detectChatTemplate(layers []*layerGGML) ([]*layerGGML, error) {
	for _, layer := range layers {
		if s := layer.GGML.KV().ChatTemplate(); s != "" {
Michael Yang's avatar
Michael Yang committed
263
			if t, err := template.Named(s); err != nil {
264
265
266
267
268
269
270
271
272
273
274
275
276
				slog.Debug("template detection", "error", err)
			} else {
				tmpl, err := NewLayer(t.Reader(), "application/vnd.ollama.image.template")
				if err != nil {
					return nil, err
				}

				tmpl.status = fmt.Sprintf("using autodetected template %s", t.Name)
				layers = append(layers, &layerGGML{tmpl, nil})
			}
		}
	}

Michael Yang's avatar
Michael Yang committed
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
	return layers, nil
}

func detectContentType(r io.Reader) (string, error) {
	var b bytes.Buffer
	if _, err := io.Copy(&b, r); err != nil {
		return "", err
	}

	if contentType := llm.DetectGGMLType(b.Bytes()); contentType != "" {
		return contentType, nil
	}

	if contentType := http.DetectContentType(b.Bytes()); contentType != "application/octet-stream" {
		return contentType, nil
	}

	return "unknown", nil
}
Michael Yang's avatar
tools  
Michael Yang committed
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313

// parseToolCalls attempts to parse a JSON string into a slice of ToolCalls.
// mxyng: this only really works if the input contains tool calls in some JSON format
func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) {
	// create a subtree from the node that ranges over .ToolCalls
	tmpl := m.Template.Subtree(func(n parse.Node) bool {
		if t, ok := n.(*parse.RangeNode); ok {
			return slices.Contains(template.Identifiers(t.Pipe), "ToolCalls")
		}

		return false
	})

	if tmpl == nil {
		return nil, false
	}

	var b bytes.Buffer
314
	if err := tmpl.Execute(&b, map[string][]api.ToolCall{
Michael Yang's avatar
tools  
Michael Yang committed
315
316
		"ToolCalls": {
			{
317
318
319
320
321
				Function: api.ToolCallFunction{
					Name: "@@name@@",
					Arguments: api.ToolCallFunctionArguments{
						"@@argument@@": 1,
					},
Michael Yang's avatar
tools  
Michael Yang committed
322
323
324
325
326
327
328
				},
			},
		},
	}); err != nil {
		return nil, false
	}

329
	var kv map[string]any
Michael Yang's avatar
tools  
Michael Yang committed
330
	// execute the subtree with placeholders to identify the keys
Michael Yang's avatar
Michael Yang committed
331
332
	// trim any commands that might exist in the template
	if err := json.Unmarshal(bytes.TrimSuffix(b.Bytes(), []byte(",")), &kv); err != nil {
Michael Yang's avatar
tools  
Michael Yang committed
333
334
335
336
337
338
		return nil, false
	}

	// find the keys that correspond to the name and arguments fields
	var name, arguments string
	for k, v := range kv {
339
340
		switch v.(type) {
		case string:
Michael Yang's avatar
tools  
Michael Yang committed
341
			name = k
342
		case map[string]any:
Michael Yang's avatar
tools  
Michael Yang committed
343
344
345
346
			arguments = k
		}
	}

347
348
349
350
	if name == "" || arguments == "" {
		return nil, false
	}

Michael Yang's avatar
Michael Yang committed
351
352
	var objs []map[string]any
	for offset := 0; offset < len(s); {
353
354
		var obj map[string]any
		decoder := json.NewDecoder(strings.NewReader(s[offset:]))
Michael Yang's avatar
Michael Yang committed
355
		if err := decoder.Decode(&obj); errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
Michael Yang's avatar
Michael Yang committed
356
357
358
359
360
361
362
363
			break
		} else if syntax := &(json.SyntaxError{}); errors.As(err, &syntax) {
			// skip over any syntax errors
			offset += int(syntax.Offset)
		} else if unmarshalType := &(json.UnmarshalTypeError{}); errors.As(err, &unmarshalType) {
			// skip over any unmarshalable types
			offset += int(unmarshalType.Offset)
		} else if err != nil {
Michael Yang's avatar
Michael Yang committed
364
			slog.Error("parseToolCalls", "error", err)
Michael Yang's avatar
tools  
Michael Yang committed
365
			return nil, false
Michael Yang's avatar
Michael Yang committed
366
		} else {
367
			offset += int(decoder.InputOffset())
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386

			// collect all nested objects
			var collect func(any) []map[string]any
			collect = func(obj any) (all []map[string]any) {
				switch o := obj.(type) {
				case map[string]any:
					all = append(all, o)
					for _, v := range o {
						all = append(all, collect(v)...)
					}
				case []any:
					for _, v := range o {
						all = append(all, collect(v)...)
					}
				}

				return all
			}
			objs = append(objs, collect(obj)...)
Michael Yang's avatar
tools  
Michael Yang committed
387
388
389
390
		}
	}

	var toolCalls []api.ToolCall
Michael Yang's avatar
Michael Yang committed
391
	for _, kv := range objs {
392
393
394
395
396
397
398
399
400
		n, nok := kv[name].(string)
		a, aok := kv[arguments].(map[string]any)
		if nok && aok {
			toolCalls = append(toolCalls, api.ToolCall{
				Function: api.ToolCallFunction{
					Name:      n,
					Arguments: a,
				},
			})
Michael Yang's avatar
tools  
Michael Yang committed
401
402
403
		}
	}

Michael Yang's avatar
Michael Yang committed
404
	return toolCalls, len(toolCalls) > 0
Michael Yang's avatar
tools  
Michael Yang committed
405
}