types.go 9.99 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
package api

3
import (
Michael Yang's avatar
Michael Yang committed
4
	"encoding/json"
5
	"fmt"
6
	"log"
Michael Yang's avatar
Michael Yang committed
7
	"math"
8
	"os"
9
10
	"reflect"
	"strings"
11
12
	"time"
)
Michael Yang's avatar
Michael Yang committed
13

Patrick Devine's avatar
Patrick Devine committed
14
type StatusError struct {
15
16
17
	StatusCode   int
	Status       string
	ErrorMessage string `json:"error"`
Patrick Devine's avatar
Patrick Devine committed
18
19
20
}

func (e StatusError) Error() string {
21
22
23
24
25
26
27
28
29
30
	switch {
	case e.Status != "" && e.ErrorMessage != "":
		return fmt.Sprintf("%s: %s", e.Status, e.ErrorMessage)
	case e.Status != "":
		return e.Status
	case e.ErrorMessage != "":
		return e.ErrorMessage
	default:
		// this should not happen
		return "something went wrong, please see the ollama server logs for details"
Patrick Devine's avatar
Patrick Devine committed
31
32
33
	}
}

34
type GenerateRequest struct {
35
36
37
38
39
	Model    string `json:"model"`
	Prompt   string `json:"prompt"`
	System   string `json:"system"`
	Template string `json:"template"`
	Context  []int  `json:"context,omitempty"`
40
	Stream   *bool  `json:"stream,omitempty"`
41

42
	Options map[string]interface{} `json:"options"`
43
44
}

Bruce MacDonald's avatar
Bruce MacDonald committed
45
46
47
48
49
50
51
52
53
54
55
type EmbeddingRequest struct {
	Model  string `json:"model"`
	Prompt string `json:"prompt"`

	Options map[string]interface{} `json:"options"`
}

type EmbeddingResponse struct {
	Embedding []float64 `json:"embedding"`
}

56
type CreateRequest struct {
57
58
59
	Name   string `json:"name"`
	Path   string `json:"path"`
	Stream *bool  `json:"stream,omitempty"`
60
61
}

62
63
64
65
type DeleteRequest struct {
	Name string `json:"name"`
}

Patrick Devine's avatar
Patrick Devine committed
66
67
68
69
70
71
72
73
74
75
76
77
type ShowRequest struct {
	Name string `json:"name"`
}

type ShowResponse struct {
	License    string `json:"license,omitempty"`
	Modelfile  string `json:"modelfile,omitempty"`
	Parameters string `json:"parameters,omitempty"`
	Template   string `json:"template,omitempty"`
	System     string `json:"system,omitempty"`
}

Patrick Devine's avatar
Patrick Devine committed
78
79
80
81
82
type CopyRequest struct {
	Source      string `json:"source"`
	Destination string `json:"destination"`
}

Bruce MacDonald's avatar
Bruce MacDonald committed
83
type PullRequest struct {
84
	Name     string `json:"name"`
85
	Insecure bool   `json:"insecure,omitempty"`
86
87
88
	Username string `json:"username"`
	Password string `json:"password"`
	Stream   *bool  `json:"stream,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
89
90
}

91
type ProgressResponse struct {
92
93
	Status    string `json:"status"`
	Digest    string `json:"digest,omitempty"`
Michael Yang's avatar
Michael Yang committed
94
95
	Total     int64  `json:"total,omitempty"`
	Completed int64  `json:"completed,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
96
97
}

98
99
type PushRequest struct {
	Name     string `json:"name"`
100
	Insecure bool   `json:"insecure,omitempty"`
101
102
103
	Username string `json:"username"`
	Password string `json:"password"`
	Stream   *bool  `json:"stream,omitempty"`
104
}
105

Patrick Devine's avatar
Patrick Devine committed
106
type ListResponse struct {
107
	Models []ModelResponse `json:"models"`
Patrick Devine's avatar
Patrick Devine committed
108
109
}

110
type ModelResponse struct {
Patrick Devine's avatar
Patrick Devine committed
111
112
	Name       string    `json:"name"`
	ModifiedAt time.Time `json:"modified_at"`
Michael Yang's avatar
Michael Yang committed
113
	Size       int64     `json:"size"`
Patrick Devine's avatar
Patrick Devine committed
114
	Digest     string    `json:"digest"`
Patrick Devine's avatar
Patrick Devine committed
115
116
}

Patrick Devine's avatar
Patrick Devine committed
117
118
119
120
type TokenResponse struct {
	Token string `json:"token"`
}

Michael Yang's avatar
Michael Yang committed
121
type GenerateResponse struct {
122
123
	Model     string    `json:"model"`
	CreatedAt time.Time `json:"created_at"`
124
	Response  string    `json:"response"`
125

Michael Yang's avatar
Michael Yang committed
126
127
	Done    bool  `json:"done"`
	Context []int `json:"context,omitempty"`
128
129

	TotalDuration      time.Duration `json:"total_duration,omitempty"`
Michael Yang's avatar
Michael Yang committed
130
	LoadDuration       time.Duration `json:"load_duration,omitempty"`
131
132
133
134
135
136
137
138
139
140
141
	PromptEvalCount    int           `json:"prompt_eval_count,omitempty"`
	PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
	EvalCount          int           `json:"eval_count,omitempty"`
	EvalDuration       time.Duration `json:"eval_duration,omitempty"`
}

func (r *GenerateResponse) Summary() {
	if r.TotalDuration > 0 {
		fmt.Fprintf(os.Stderr, "total duration:       %v\n", r.TotalDuration)
	}

Michael Yang's avatar
Michael Yang committed
142
143
144
145
	if r.LoadDuration > 0 {
		fmt.Fprintf(os.Stderr, "load duration:        %v\n", r.LoadDuration)
	}

146
147
148
149
150
151
152
153
154
155
156
157
158
159
	if r.PromptEvalCount > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval count:    %d token(s)\n", r.PromptEvalCount)
	}

	if r.PromptEvalDuration > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval duration: %s\n", r.PromptEvalDuration)
		fmt.Fprintf(os.Stderr, "prompt eval rate:     %.2f tokens/s\n", float64(r.PromptEvalCount)/r.PromptEvalDuration.Seconds())
	}

	if r.EvalCount > 0 {
		fmt.Fprintf(os.Stderr, "eval count:           %d token(s)\n", r.EvalCount)
	}

	if r.EvalDuration > 0 {
Michael Yang's avatar
Michael Yang committed
160
		fmt.Fprintf(os.Stderr, "eval duration:        %s\n", r.EvalDuration)
161
162
		fmt.Fprintf(os.Stderr, "eval rate:            %.2f tokens/s\n", float64(r.EvalCount)/r.EvalDuration.Seconds())
	}
163
164
}

Michael Yang's avatar
Michael Yang committed
165
166
type Options struct {
	Seed int `json:"seed,omitempty"`
167

Michael Yang's avatar
Michael Yang committed
168
169
170
171
	// Backend options
	UseNUMA bool `json:"numa,omitempty"`

	// Model options
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
	NumCtx             int     `json:"num_ctx,omitempty"`
	NumKeep            int     `json:"num_keep,omitempty"`
	NumBatch           int     `json:"num_batch,omitempty"`
	NumGQA             int     `json:"num_gqa,omitempty"`
	NumGPU             int     `json:"num_gpu,omitempty"`
	MainGPU            int     `json:"main_gpu,omitempty"`
	LowVRAM            bool    `json:"low_vram,omitempty"`
	F16KV              bool    `json:"f16_kv,omitempty"`
	LogitsAll          bool    `json:"logits_all,omitempty"`
	VocabOnly          bool    `json:"vocab_only,omitempty"`
	UseMMap            bool    `json:"use_mmap,omitempty"`
	UseMLock           bool    `json:"use_mlock,omitempty"`
	EmbeddingOnly      bool    `json:"embedding_only,omitempty"`
	RopeFrequencyBase  float32 `json:"rope_frequency_base,omitempty"`
	RopeFrequencyScale float32 `json:"rope_frequency_scale,omitempty"`
Michael Yang's avatar
Michael Yang committed
187
188

	// Predict options
189
	NumPredict       int      `json:"num_predict,omitempty"`
Michael Yang's avatar
Michael Yang committed
190
191
192
193
	TopK             int      `json:"top_k,omitempty"`
	TopP             float32  `json:"top_p,omitempty"`
	TFSZ             float32  `json:"tfs_z,omitempty"`
	TypicalP         float32  `json:"typical_p,omitempty"`
194
195
196
197
198
	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
	Temperature      float32  `json:"temperature,omitempty"`
	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
Michael Yang's avatar
Michael Yang committed
199
200
201
202
	Mirostat         int      `json:"mirostat,omitempty"`
	MirostatTau      float32  `json:"mirostat_tau,omitempty"`
	MirostatEta      float32  `json:"mirostat_eta,omitempty"`
	PenalizeNewline  bool     `json:"penalize_newline,omitempty"`
203
	Stop             []string `json:"stop,omitempty"`
204

Michael Yang's avatar
Michael Yang committed
205
	NumThread int `json:"num_thread,omitempty"`
Jeffrey Morgan's avatar
Jeffrey Morgan committed
206
207
}

208
209
var ErrInvalidOpts = fmt.Errorf("invalid options")

210
211
212
213
214
215
216
217
218
219
220
221
222
func (opts *Options) FromMap(m map[string]interface{}) error {
	valueOpts := reflect.ValueOf(opts).Elem() // names of the fields in the options struct
	typeOpts := reflect.TypeOf(opts).Elem()   // types of the fields in the options struct

	// build map of json struct tags to their types
	jsonOpts := make(map[string]reflect.StructField)
	for _, field := range reflect.VisibleFields(typeOpts) {
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
		if jsonTag != "" {
			jsonOpts[jsonTag] = field
		}
	}

223
	invalidOpts := []string{}
224
225
226
227
	for key, val := range m {
		if opt, ok := jsonOpts[key]; ok {
			field := valueOpts.FieldByName(opt.Name)
			if field.IsValid() && field.CanSet() {
Michael Yang's avatar
Michael Yang committed
228
229
230
231
				if val == nil {
					continue
				}

232
233
				switch field.Kind() {
				case reflect.Int:
Michael Yang's avatar
Michael Yang committed
234
235
236
237
238
239
240
					switch t := val.(type) {
					case int64:
						field.SetInt(t)
					case float64:
						// when JSON unmarshals numbers, it uses float64, not int
						field.SetInt(int64(t))
					default:
241
						log.Printf("could not convert model parameter %v of type %T to int, skipped", key, val)
242
243
244
245
					}
				case reflect.Bool:
					val, ok := val.(bool)
					if !ok {
246
						log.Printf("could not convert model parameter %v of type %T to bool, skipped", key, val)
247
248
249
250
251
252
253
						continue
					}
					field.SetBool(val)
				case reflect.Float32:
					// JSON unmarshals to float64
					val, ok := val.(float64)
					if !ok {
254
						log.Printf("could not convert model parameter %v of type %T to float32, skipped", key, val)
255
256
257
258
259
260
						continue
					}
					field.SetFloat(val)
				case reflect.String:
					val, ok := val.(string)
					if !ok {
261
						log.Printf("could not convert model parameter %v of type %T to string, skipped", key, val)
262
263
264
265
266
267
268
						continue
					}
					field.SetString(val)
				case reflect.Slice:
					// JSON unmarshals to []interface{}, not []string
					val, ok := val.([]interface{})
					if !ok {
269
						log.Printf("could not convert model parameter %v of type %T to slice, skipped", key, val)
270
271
272
273
274
275
276
						continue
					}
					// convert []interface{} to []string
					slice := make([]string, len(val))
					for i, item := range val {
						str, ok := item.(string)
						if !ok {
277
							log.Printf("could not convert model parameter %v of type %T to slice of strings, skipped", key, item)
278
279
280
281
282
283
284
285
286
							continue
						}
						slice[i] = str
					}
					field.Set(reflect.ValueOf(slice))
				default:
					return fmt.Errorf("unknown type loading config params: %v", field.Kind())
				}
			}
287
288
		} else {
			invalidOpts = append(invalidOpts, key)
289
290
		}
	}
291
292
293
294

	if len(invalidOpts) > 0 {
		return fmt.Errorf("%w: %v", ErrInvalidOpts, strings.Join(invalidOpts, ", "))
	}
295
296
297
	return nil
}

Michael Yang's avatar
Michael Yang committed
298
299
func DefaultOptions() Options {
	return Options{
300
301
302
		// options set on request to runner
		NumPredict:       -1,
		NumKeep:          -1,
Michael Yang's avatar
Michael Yang committed
303
304
305
306
307
		Temperature:      0.8,
		TopK:             40,
		TopP:             0.9,
		TFSZ:             1.0,
		TypicalP:         1.0,
308
309
310
311
		RepeatLastN:      64,
		RepeatPenalty:    1.1,
		PresencePenalty:  0.0,
		FrequencyPenalty: 0.0,
Michael Yang's avatar
Michael Yang committed
312
313
314
		Mirostat:         0,
		MirostatTau:      5.0,
		MirostatEta:      0.1,
Michael Yang's avatar
Michael Yang committed
315
		PenalizeNewline:  true,
316
		Seed:             -1,
Michael Yang's avatar
Michael Yang committed
317

318
319
320
321
322
323
324
325
326
327
328
329
330
331
		// options set when the model is loaded
		NumCtx:             2048,
		RopeFrequencyBase:  10000.0,
		RopeFrequencyScale: 1.0,
		NumBatch:           512,
		NumGPU:             -1, // -1 here indicates that NumGPU should be set dynamically
		NumGQA:             1,
		NumThread:          0, // let the runtime decide
		LowVRAM:            false,
		F16KV:              true,
		UseMLock:           false,
		UseMMap:            true,
		UseNUMA:            false,
		EmbeddingOnly:      true,
Michael Yang's avatar
Michael Yang committed
332
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
333
}
Michael Yang's avatar
Michael Yang committed
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362

type Duration struct {
	time.Duration
}

func (d *Duration) UnmarshalJSON(b []byte) (err error) {
	var v any
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	d.Duration = 5 * time.Minute

	switch t := v.(type) {
	case float64:
		if t < 0 {
			t = math.MaxFloat64
		}

		d.Duration = time.Duration(t)
	case string:
		d.Duration, err = time.ParseDuration(t)
		if err != nil {
			return err
		}
	}

	return nil
}