model.go 9.14 KB
Newer Older
mashun1's avatar
v1  
mashun1 committed
1
2
3
4
5
6
package server

import (
	"archive/zip"
	"bytes"
	"context"
xuxzh1's avatar
init  
xuxzh1 committed
7
	"encoding/json"
mashun1's avatar
v1  
mashun1 committed
8
9
10
	"errors"
	"fmt"
	"io"
xuxzh1's avatar
init  
xuxzh1 committed
11
	"log/slog"
mashun1's avatar
v1  
mashun1 committed
12
13
14
	"net/http"
	"os"
	"path/filepath"
xuxzh1's avatar
init  
xuxzh1 committed
15
16
17
	"slices"
	"strings"
	"text/template/parse"
mashun1's avatar
v1  
mashun1 committed
18
19
20
21

	"github.com/ollama/ollama/api"
	"github.com/ollama/ollama/convert"
	"github.com/ollama/ollama/llm"
xuxzh1's avatar
init  
xuxzh1 committed
22
	"github.com/ollama/ollama/template"
mashun1's avatar
v1  
mashun1 committed
23
24
25
26
27
	"github.com/ollama/ollama/types/model"
)

var intermediateBlobs map[string]string = make(map[string]string)

xuxzh1's avatar
init  
xuxzh1 committed
28
29
type layerGGML struct {
	Layer
mashun1's avatar
v1  
mashun1 committed
30
31
32
	*llm.GGML
}

xuxzh1's avatar
init  
xuxzh1 committed
33
34
func parseFromModel(ctx context.Context, name model.Name, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
	m, err := ParseNamedManifest(name)
mashun1's avatar
v1  
mashun1 committed
35
36
37
38
39
40
	switch {
	case errors.Is(err, os.ErrNotExist):
		if err := PullModel(ctx, name.String(), &registryOptions{}, fn); err != nil {
			return nil, err
		}

xuxzh1's avatar
init  
xuxzh1 committed
41
		m, err = ParseNamedManifest(name)
mashun1's avatar
v1  
mashun1 committed
42
43
44
45
46
47
48
		if err != nil {
			return nil, err
		}
	case err != nil:
		return nil, err
	}

xuxzh1's avatar
init  
xuxzh1 committed
49
50
	for _, layer := range m.Layers {
		layer, err := NewLayerFromLayer(layer.Digest, layer.MediaType, name.DisplayShortest())
mashun1's avatar
v1  
mashun1 committed
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
		if err != nil {
			return nil, err
		}

		switch layer.MediaType {
		case "application/vnd.ollama.image.model",
			"application/vnd.ollama.image.projector",
			"application/vnd.ollama.image.adapter":
			blobpath, err := GetBlobsPath(layer.Digest)
			if err != nil {
				return nil, err
			}

			blob, err := os.Open(blobpath)
			if err != nil {
				return nil, err
			}
			defer blob.Close()

xuxzh1's avatar
init  
xuxzh1 committed
70
			ggml, _, err := llm.DecodeGGML(blob, 0)
mashun1's avatar
v1  
mashun1 committed
71
72
73
74
			if err != nil {
				return nil, err
			}

xuxzh1's avatar
init  
xuxzh1 committed
75
			layers = append(layers, &layerGGML{layer, ggml})
mashun1's avatar
v1  
mashun1 committed
76
		default:
xuxzh1's avatar
init  
xuxzh1 committed
77
			layers = append(layers, &layerGGML{layer, nil})
mashun1's avatar
v1  
mashun1 committed
78
79
80
81
82
83
		}
	}

	return layers, nil
}

xuxzh1's avatar
update  
xuxzh1 committed
84
func parseFromZipFile(_ context.Context, command string, baseLayers []*layerGGML, f *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
xuxzh1's avatar
init  
xuxzh1 committed
85
	fi, err := f.Stat()
mashun1's avatar
v1  
mashun1 committed
86
87
88
89
	if err != nil {
		return nil, err
	}

xuxzh1's avatar
init  
xuxzh1 committed
90
	r, err := zip.NewReader(f, fi.Size())
mashun1's avatar
v1  
mashun1 committed
91
92
93
94
	if err != nil {
		return nil, err
	}

xuxzh1's avatar
init  
xuxzh1 committed
95
	p, err := os.MkdirTemp(filepath.Dir(f.Name()), "")
mashun1's avatar
v1  
mashun1 committed
96
97
98
	if err != nil {
		return nil, err
	}
xuxzh1's avatar
init  
xuxzh1 committed
99
	defer os.RemoveAll(p)
mashun1's avatar
v1  
mashun1 committed
100
101
102
103

	fn(api.ProgressResponse{Status: "converting model"})
	// TODO(mxyng): this should write directly into a layer
	// e.g. NewLayer(arch.Reader(), "application/vnd.ollama.image.model")
xuxzh1's avatar
init  
xuxzh1 committed
104
	t, err := os.CreateTemp(p, "fp16")
mashun1's avatar
v1  
mashun1 committed
105
106
107
	if err != nil {
		return nil, err
	}
xuxzh1's avatar
init  
xuxzh1 committed
108
109
	defer t.Close()
	defer os.Remove(t.Name())
mashun1's avatar
v1  
mashun1 committed
110

xuxzh1's avatar
update  
xuxzh1 committed
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
	var layerType string

	switch command {
	case "adapter":
		var baseModel *llm.GGML
		for _, l := range baseLayers {
			if l.GGML != nil {
				baseModel = l.GGML
				break
			}
		}

		if baseModel == nil {
			return nil, fmt.Errorf("no base model specified for the adapter")
		}

		if err := convert.ConvertAdapter(convert.NewZipReader(r, p, 32<<20), t, baseModel.KV()); err != nil {
			return nil, err
		}
		layerType = "application/vnd.ollama.image.adapter"
	case "model":
		if err := convert.ConvertModel(convert.NewZipReader(r, p, 32<<20), t); err != nil {
			return nil, err
		}
		layerType = "application/vnd.ollama.image.model"
mashun1's avatar
v1  
mashun1 committed
136
137
	}

xuxzh1's avatar
init  
xuxzh1 committed
138
	if _, err := t.Seek(0, io.SeekStart); err != nil {
mashun1's avatar
v1  
mashun1 committed
139
140
141
		return nil, err
	}

xuxzh1's avatar
update  
xuxzh1 committed
142
	layer, err := NewLayer(t, layerType)
mashun1's avatar
v1  
mashun1 committed
143
144
145
146
147
148
149
150
151
152
	if err != nil {
		return nil, err
	}

	bin, err := layer.Open()
	if err != nil {
		return nil, err
	}
	defer bin.Close()

xuxzh1's avatar
init  
xuxzh1 committed
153
	ggml, _, err := llm.DecodeGGML(bin, 0)
mashun1's avatar
v1  
mashun1 committed
154
155
156
157
	if err != nil {
		return nil, err
	}

xuxzh1's avatar
init  
xuxzh1 committed
158
	layers = append(layers, &layerGGML{layer, ggml})
mashun1's avatar
v1  
mashun1 committed
159
160

	intermediateBlobs[digest] = layer.Digest
xuxzh1's avatar
init  
xuxzh1 committed
161
	return detectChatTemplate(layers)
mashun1's avatar
v1  
mashun1 committed
162
163
}

xuxzh1's avatar
update  
xuxzh1 committed
164
func parseFromFile(ctx context.Context, command string, baseLayers []*layerGGML, file *os.File, digest string, fn func(api.ProgressResponse)) (layers []*layerGGML, err error) {
mashun1's avatar
v1  
mashun1 committed
165
166
167
168
169
170
171
172
173
174
	sr := io.NewSectionReader(file, 0, 512)
	contentType, err := detectContentType(sr)
	if err != nil {
		return nil, err
	}

	switch contentType {
	case "gguf", "ggla":
		// noop
	case "application/zip":
xuxzh1's avatar
update  
xuxzh1 committed
175
		return parseFromZipFile(ctx, command, baseLayers, file, digest, fn)
mashun1's avatar
v1  
mashun1 committed
176
177
178
179
180
181
182
183
184
185
186
	default:
		return nil, fmt.Errorf("unsupported content type: %s", contentType)
	}

	stat, err := file.Stat()
	if err != nil {
		return nil, err
	}

	var offset int64
	for offset < stat.Size() {
xuxzh1's avatar
init  
xuxzh1 committed
187
		ggml, n, err := llm.DecodeGGML(file, 0)
mashun1's avatar
v1  
mashun1 committed
188
189
190
191
192
193
194
		if errors.Is(err, io.EOF) {
			break
		} else if err != nil {
			return nil, err
		}

		mediatype := "application/vnd.ollama.image.model"
xuxzh1's avatar
update  
xuxzh1 committed
195
		if ggml.Name() == "ggla" || ggml.KV().Kind() == "adapter" {
mashun1's avatar
v1  
mashun1 committed
196
			mediatype = "application/vnd.ollama.image.adapter"
xuxzh1's avatar
update  
xuxzh1 committed
197
198
199
		}

		if _, ok := ggml.KV()[fmt.Sprintf("%s.vision.block_count", ggml.KV().Architecture())]; ok || ggml.KV().Kind() == "projector" {
mashun1's avatar
v1  
mashun1 committed
200
201
202
			mediatype = "application/vnd.ollama.image.projector"
		}

xuxzh1's avatar
update  
xuxzh1 committed
203
204
205
206
207
208
209
210
211
212
213
214
215
216
		var layer Layer
		if digest != "" && n == stat.Size() && offset == 0 {
			layer, err = NewLayerFromLayer(digest, mediatype, file.Name())
			if err != nil {
				slog.Debug("could not create new layer from layer", "error", err)
			}
		}

		// Fallback to creating layer from file copy (either NewLayerFromLayer failed, or digest empty/n != stat.Size())
		if layer.Digest == "" {
			layer, err = NewLayer(io.NewSectionReader(file, offset, n), mediatype)
			if err != nil {
				return nil, err
			}
mashun1's avatar
v1  
mashun1 committed
217
218
		}

xuxzh1's avatar
init  
xuxzh1 committed
219
		layers = append(layers, &layerGGML{layer, ggml})
mashun1's avatar
v1  
mashun1 committed
220
221
222
		offset = n
	}

xuxzh1's avatar
init  
xuxzh1 committed
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
	return detectChatTemplate(layers)
}

func detectChatTemplate(layers []*layerGGML) ([]*layerGGML, error) {
	for _, layer := range layers {
		if s := layer.GGML.KV().ChatTemplate(); s != "" {
			if t, err := template.Named(s); err != nil {
				slog.Debug("template detection", "error", err)
			} else {
				layer, err := NewLayer(t.Reader(), "application/vnd.ollama.image.template")
				if err != nil {
					return nil, err
				}

				layer.status = fmt.Sprintf("using autodetected template %s", t.Name)
				layers = append(layers, &layerGGML{layer, nil})

				if t.Parameters != nil {
					var b bytes.Buffer
					if err := json.NewEncoder(&b).Encode(t.Parameters); err != nil {
						return nil, err
					}

					layer, err := NewLayer(&b, "application/vnd.ollama.image.params")
					if err != nil {
						return nil, err
					}

					layers = append(layers, &layerGGML{layer, nil})
				}
			}
		}
	}

mashun1's avatar
v1  
mashun1 committed
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
	return layers, nil
}

func detectContentType(r io.Reader) (string, error) {
	var b bytes.Buffer
	if _, err := io.Copy(&b, r); err != nil {
		return "", err
	}

	if contentType := llm.DetectGGMLType(b.Bytes()); contentType != "" {
		return contentType, nil
	}

	if contentType := http.DetectContentType(b.Bytes()); contentType != "application/octet-stream" {
		return contentType, nil
	}

	return "unknown", nil
}
xuxzh1's avatar
init  
xuxzh1 committed
276

xuxzh1's avatar
update  
xuxzh1 committed
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
func parseObjects(s string) []map[string]any {
	var objs []map[string]any
	for offset := 0; offset < len(s); {
		var obj map[string]any
		decoder := json.NewDecoder(strings.NewReader(s[offset:]))
		if err := decoder.Decode(&obj); errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
			break
		} else if syntax := &(json.SyntaxError{}); errors.As(err, &syntax) {
			// skip over any syntax errors
			offset += int(syntax.Offset)
		} else if unmarshalType := &(json.UnmarshalTypeError{}); errors.As(err, &unmarshalType) {
			// skip over any unmarshalable types
			offset += int(unmarshalType.Offset)
		} else if err != nil {
			return nil
		} else {
			offset += int(decoder.InputOffset())
			objs = append(objs, obj)
		}
	}

	return objs
}

xuxzh1's avatar
init  
xuxzh1 committed
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
// parseToolCalls attempts to parse a JSON string into a slice of ToolCalls.
// mxyng: this only really works if the input contains tool calls in some JSON format
func (m *Model) parseToolCalls(s string) ([]api.ToolCall, bool) {
	// create a subtree from the node that ranges over .ToolCalls
	tmpl := m.Template.Subtree(func(n parse.Node) bool {
		if t, ok := n.(*parse.RangeNode); ok {
			return slices.Contains(template.Identifiers(t.Pipe), "ToolCalls")
		}

		return false
	})

	if tmpl == nil {
		return nil, false
	}

	var b bytes.Buffer
	if err := tmpl.Execute(&b, map[string][]api.ToolCall{
		"ToolCalls": {
			{
				Function: api.ToolCallFunction{
					Name: "@@name@@",
					Arguments: api.ToolCallFunctionArguments{
						"@@argument@@": 1,
					},
				},
			},
		},
	}); err != nil {
		return nil, false
	}

xuxzh1's avatar
update  
xuxzh1 committed
333
334
	templateObjects := parseObjects(b.String())
	if len(templateObjects) == 0 {
xuxzh1's avatar
init  
xuxzh1 committed
335
336
337
338
339
		return nil, false
	}

	// find the keys that correspond to the name and arguments fields
	var name, arguments string
xuxzh1's avatar
update  
xuxzh1 committed
340
	for k, v := range templateObjects[0] {
xuxzh1's avatar
init  
xuxzh1 committed
341
342
343
344
345
346
347
348
349
350
351
352
		switch v.(type) {
		case string:
			name = k
		case map[string]any:
			arguments = k
		}
	}

	if name == "" || arguments == "" {
		return nil, false
	}

xuxzh1's avatar
update  
xuxzh1 committed
353
354
355
356
	responseObjects := parseObjects(s)
	if len(responseObjects) == 0 {
		return nil, false
	}
xuxzh1's avatar
init  
xuxzh1 committed
357

xuxzh1's avatar
update  
xuxzh1 committed
358
359
360
361
362
363
364
365
366
367
368
369
	// collect all nested objects
	var collect func(any) []map[string]any
	collect = func(obj any) (all []map[string]any) {
		switch o := obj.(type) {
		case map[string]any:
			all = append(all, o)
			for _, v := range o {
				all = append(all, collect(v)...)
			}
		case []any:
			for _, v := range o {
				all = append(all, collect(v)...)
xuxzh1's avatar
init  
xuxzh1 committed
370
371
			}
		}
xuxzh1's avatar
update  
xuxzh1 committed
372
373
374
375
376
377
378

		return all
	}

	var objs []map[string]any
	for _, p := range responseObjects {
		objs = append(objs, collect(p)...)
xuxzh1's avatar
init  
xuxzh1 committed
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
	}

	var toolCalls []api.ToolCall
	for _, kv := range objs {
		n, nok := kv[name].(string)
		a, aok := kv[arguments].(map[string]any)
		if nok && aok {
			toolCalls = append(toolCalls, api.ToolCall{
				Function: api.ToolCallFunction{
					Name:      n,
					Arguments: a,
				},
			})
		}
	}

	return toolCalls, len(toolCalls) > 0
}