types.go 9.75 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
package api

3
import (
Michael Yang's avatar
Michael Yang committed
4
	"encoding/json"
5
	"fmt"
6
	"log"
Michael Yang's avatar
Michael Yang committed
7
	"math"
8
	"os"
9
10
	"reflect"
	"strings"
11
12
	"time"
)
Michael Yang's avatar
Michael Yang committed
13

Patrick Devine's avatar
Patrick Devine committed
14
type StatusError struct {
15
16
17
	StatusCode   int
	Status       string
	ErrorMessage string `json:"error"`
Patrick Devine's avatar
Patrick Devine committed
18
19
20
}

func (e StatusError) Error() string {
21
22
23
24
25
26
27
28
29
30
	switch {
	case e.Status != "" && e.ErrorMessage != "":
		return fmt.Sprintf("%s: %s", e.Status, e.ErrorMessage)
	case e.Status != "":
		return e.Status
	case e.ErrorMessage != "":
		return e.ErrorMessage
	default:
		// this should not happen
		return "something went wrong, please see the ollama server logs for details"
Patrick Devine's avatar
Patrick Devine committed
31
32
33
	}
}

34
type GenerateRequest struct {
35
36
37
38
39
	Model    string `json:"model"`
	Prompt   string `json:"prompt"`
	System   string `json:"system"`
	Template string `json:"template"`
	Context  []int  `json:"context,omitempty"`
40
	Stream   *bool  `json:"stream,omitempty"`
41

42
	Options map[string]interface{} `json:"options"`
43
44
}

Bruce MacDonald's avatar
Bruce MacDonald committed
45
46
47
48
49
50
51
52
53
54
55
type EmbeddingRequest struct {
	Model  string `json:"model"`
	Prompt string `json:"prompt"`

	Options map[string]interface{} `json:"options"`
}

type EmbeddingResponse struct {
	Embedding []float64 `json:"embedding"`
}

56
type CreateRequest struct {
57
58
59
	Name   string `json:"name"`
	Path   string `json:"path"`
	Stream *bool  `json:"stream,omitempty"`
60
61
}

62
63
64
65
type DeleteRequest struct {
	Name string `json:"name"`
}

Patrick Devine's avatar
Patrick Devine committed
66
67
68
69
70
71
72
73
74
75
76
77
type ShowRequest struct {
	Name string `json:"name"`
}

type ShowResponse struct {
	License    string `json:"license,omitempty"`
	Modelfile  string `json:"modelfile,omitempty"`
	Parameters string `json:"parameters,omitempty"`
	Template   string `json:"template,omitempty"`
	System     string `json:"system,omitempty"`
}

Patrick Devine's avatar
Patrick Devine committed
78
79
80
81
82
type CopyRequest struct {
	Source      string `json:"source"`
	Destination string `json:"destination"`
}

Bruce MacDonald's avatar
Bruce MacDonald committed
83
type PullRequest struct {
84
	Name     string `json:"name"`
85
	Insecure bool   `json:"insecure,omitempty"`
86
87
88
	Username string `json:"username"`
	Password string `json:"password"`
	Stream   *bool  `json:"stream,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
89
90
}

91
type ProgressResponse struct {
92
93
	Status    string `json:"status"`
	Digest    string `json:"digest,omitempty"`
Michael Yang's avatar
Michael Yang committed
94
95
	Total     int64  `json:"total,omitempty"`
	Completed int64  `json:"completed,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
96
97
}

98
99
type PushRequest struct {
	Name     string `json:"name"`
100
	Insecure bool   `json:"insecure,omitempty"`
101
102
103
	Username string `json:"username"`
	Password string `json:"password"`
	Stream   *bool  `json:"stream,omitempty"`
104
}
105

Patrick Devine's avatar
Patrick Devine committed
106
type ListResponse struct {
107
	Models []ModelResponse `json:"models"`
Patrick Devine's avatar
Patrick Devine committed
108
109
}

110
type ModelResponse struct {
Patrick Devine's avatar
Patrick Devine committed
111
112
	Name       string    `json:"name"`
	ModifiedAt time.Time `json:"modified_at"`
Michael Yang's avatar
Michael Yang committed
113
	Size       int64     `json:"size"`
Patrick Devine's avatar
Patrick Devine committed
114
	Digest     string    `json:"digest"`
Patrick Devine's avatar
Patrick Devine committed
115
116
}

Patrick Devine's avatar
Patrick Devine committed
117
118
119
120
type TokenResponse struct {
	Token string `json:"token"`
}

Michael Yang's avatar
Michael Yang committed
121
type GenerateResponse struct {
122
123
	Model     string    `json:"model"`
	CreatedAt time.Time `json:"created_at"`
124
	Response  string    `json:"response"`
125

Michael Yang's avatar
Michael Yang committed
126
127
	Done    bool  `json:"done"`
	Context []int `json:"context,omitempty"`
128
129

	TotalDuration      time.Duration `json:"total_duration,omitempty"`
Michael Yang's avatar
Michael Yang committed
130
	LoadDuration       time.Duration `json:"load_duration,omitempty"`
131
132
133
134
135
136
137
138
139
140
141
	PromptEvalCount    int           `json:"prompt_eval_count,omitempty"`
	PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
	EvalCount          int           `json:"eval_count,omitempty"`
	EvalDuration       time.Duration `json:"eval_duration,omitempty"`
}

func (r *GenerateResponse) Summary() {
	if r.TotalDuration > 0 {
		fmt.Fprintf(os.Stderr, "total duration:       %v\n", r.TotalDuration)
	}

Michael Yang's avatar
Michael Yang committed
142
143
144
145
	if r.LoadDuration > 0 {
		fmt.Fprintf(os.Stderr, "load duration:        %v\n", r.LoadDuration)
	}

146
147
148
149
150
151
152
153
154
155
156
157
158
159
	if r.PromptEvalCount > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval count:    %d token(s)\n", r.PromptEvalCount)
	}

	if r.PromptEvalDuration > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval duration: %s\n", r.PromptEvalDuration)
		fmt.Fprintf(os.Stderr, "prompt eval rate:     %.2f tokens/s\n", float64(r.PromptEvalCount)/r.PromptEvalDuration.Seconds())
	}

	if r.EvalCount > 0 {
		fmt.Fprintf(os.Stderr, "eval count:           %d token(s)\n", r.EvalCount)
	}

	if r.EvalDuration > 0 {
Michael Yang's avatar
Michael Yang committed
160
		fmt.Fprintf(os.Stderr, "eval duration:        %s\n", r.EvalDuration)
161
162
		fmt.Fprintf(os.Stderr, "eval rate:            %.2f tokens/s\n", float64(r.EvalCount)/r.EvalDuration.Seconds())
	}
163
164
}

Michael Yang's avatar
Michael Yang committed
165
166
type Options struct {
	Seed int `json:"seed,omitempty"`
167

Michael Yang's avatar
Michael Yang committed
168
169
170
171
	// Backend options
	UseNUMA bool `json:"numa,omitempty"`

	// Model options
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
	NumCtx             int     `json:"num_ctx,omitempty"`
	NumKeep            int     `json:"num_keep,omitempty"`
	NumBatch           int     `json:"num_batch,omitempty"`
	NumGQA             int     `json:"num_gqa,omitempty"`
	NumGPU             int     `json:"num_gpu,omitempty"`
	MainGPU            int     `json:"main_gpu,omitempty"`
	LowVRAM            bool    `json:"low_vram,omitempty"`
	F16KV              bool    `json:"f16_kv,omitempty"`
	LogitsAll          bool    `json:"logits_all,omitempty"`
	VocabOnly          bool    `json:"vocab_only,omitempty"`
	UseMMap            bool    `json:"use_mmap,omitempty"`
	UseMLock           bool    `json:"use_mlock,omitempty"`
	EmbeddingOnly      bool    `json:"embedding_only,omitempty"`
	RopeFrequencyBase  float32 `json:"rope_frequency_base,omitempty"`
	RopeFrequencyScale float32 `json:"rope_frequency_scale,omitempty"`
Michael Yang's avatar
Michael Yang committed
187
188

	// Predict options
189
	NumPredict       int      `json:"num_predict,omitempty"`
Michael Yang's avatar
Michael Yang committed
190
191
192
193
	TopK             int      `json:"top_k,omitempty"`
	TopP             float32  `json:"top_p,omitempty"`
	TFSZ             float32  `json:"tfs_z,omitempty"`
	TypicalP         float32  `json:"typical_p,omitempty"`
194
195
196
197
198
	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
	Temperature      float32  `json:"temperature,omitempty"`
	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
Michael Yang's avatar
Michael Yang committed
199
200
201
202
	Mirostat         int      `json:"mirostat,omitempty"`
	MirostatTau      float32  `json:"mirostat_tau,omitempty"`
	MirostatEta      float32  `json:"mirostat_eta,omitempty"`
	PenalizeNewline  bool     `json:"penalize_newline,omitempty"`
203
	Stop             []string `json:"stop,omitempty"`
204

Michael Yang's avatar
Michael Yang committed
205
	NumThread int `json:"num_thread,omitempty"`
Jeffrey Morgan's avatar
Jeffrey Morgan committed
206
207
}

208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
func (opts *Options) FromMap(m map[string]interface{}) error {
	valueOpts := reflect.ValueOf(opts).Elem() // names of the fields in the options struct
	typeOpts := reflect.TypeOf(opts).Elem()   // types of the fields in the options struct

	// build map of json struct tags to their types
	jsonOpts := make(map[string]reflect.StructField)
	for _, field := range reflect.VisibleFields(typeOpts) {
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
		if jsonTag != "" {
			jsonOpts[jsonTag] = field
		}
	}

	for key, val := range m {
		if opt, ok := jsonOpts[key]; ok {
			field := valueOpts.FieldByName(opt.Name)
			if field.IsValid() && field.CanSet() {
Michael Yang's avatar
Michael Yang committed
225
226
227
228
				if val == nil {
					continue
				}

229
230
				switch field.Kind() {
				case reflect.Int:
Michael Yang's avatar
Michael Yang committed
231
232
233
234
235
236
237
					switch t := val.(type) {
					case int64:
						field.SetInt(t)
					case float64:
						// when JSON unmarshals numbers, it uses float64, not int
						field.SetInt(int64(t))
					default:
238
						log.Printf("could not convert model parameter %v of type %T to int, skipped", key, val)
239
240
241
242
					}
				case reflect.Bool:
					val, ok := val.(bool)
					if !ok {
243
						log.Printf("could not convert model parameter %v of type %T to bool, skipped", key, val)
244
245
246
247
248
249
250
						continue
					}
					field.SetBool(val)
				case reflect.Float32:
					// JSON unmarshals to float64
					val, ok := val.(float64)
					if !ok {
251
						log.Printf("could not convert model parameter %v of type %T to float32, skipped", key, val)
252
253
254
255
256
257
						continue
					}
					field.SetFloat(val)
				case reflect.String:
					val, ok := val.(string)
					if !ok {
258
						log.Printf("could not convert model parameter %v of type %T to string, skipped", key, val)
259
260
261
262
263
264
265
						continue
					}
					field.SetString(val)
				case reflect.Slice:
					// JSON unmarshals to []interface{}, not []string
					val, ok := val.([]interface{})
					if !ok {
266
						log.Printf("could not convert model parameter %v of type %T to slice, skipped", key, val)
267
268
269
270
271
272
273
						continue
					}
					// convert []interface{} to []string
					slice := make([]string, len(val))
					for i, item := range val {
						str, ok := item.(string)
						if !ok {
274
							log.Printf("could not convert model parameter %v of type %T to slice of strings, skipped", key, item)
275
276
277
278
279
280
281
282
283
284
285
286
287
288
							continue
						}
						slice[i] = str
					}
					field.Set(reflect.ValueOf(slice))
				default:
					return fmt.Errorf("unknown type loading config params: %v", field.Kind())
				}
			}
		}
	}
	return nil
}

Michael Yang's avatar
Michael Yang committed
289
290
func DefaultOptions() Options {
	return Options{
291
292
293
		// options set on request to runner
		NumPredict:       -1,
		NumKeep:          -1,
Michael Yang's avatar
Michael Yang committed
294
295
296
297
298
		Temperature:      0.8,
		TopK:             40,
		TopP:             0.9,
		TFSZ:             1.0,
		TypicalP:         1.0,
299
300
301
302
		RepeatLastN:      64,
		RepeatPenalty:    1.1,
		PresencePenalty:  0.0,
		FrequencyPenalty: 0.0,
Michael Yang's avatar
Michael Yang committed
303
304
305
		Mirostat:         0,
		MirostatTau:      5.0,
		MirostatEta:      0.1,
Michael Yang's avatar
Michael Yang committed
306
		PenalizeNewline:  true,
307
		Seed:             -1,
Michael Yang's avatar
Michael Yang committed
308

309
310
311
312
313
314
315
316
317
318
319
320
321
322
		// options set when the model is loaded
		NumCtx:             2048,
		RopeFrequencyBase:  10000.0,
		RopeFrequencyScale: 1.0,
		NumBatch:           512,
		NumGPU:             -1, // -1 here indicates that NumGPU should be set dynamically
		NumGQA:             1,
		NumThread:          0, // let the runtime decide
		LowVRAM:            false,
		F16KV:              true,
		UseMLock:           false,
		UseMMap:            true,
		UseNUMA:            false,
		EmbeddingOnly:      true,
Michael Yang's avatar
Michael Yang committed
323
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
324
}
Michael Yang's avatar
Michael Yang committed
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353

type Duration struct {
	time.Duration
}

func (d *Duration) UnmarshalJSON(b []byte) (err error) {
	var v any
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	d.Duration = 5 * time.Minute

	switch t := v.(type) {
	case float64:
		if t < 0 {
			t = math.MaxFloat64
		}

		d.Duration = time.Duration(t)
	case string:
		d.Duration, err = time.ParseDuration(t)
		if err != nil {
			return err
		}
	}

	return nil
}