types.go 21.7 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
package api

3
import (
Michael Yang's avatar
Michael Yang committed
4
	"encoding/json"
5
	"fmt"
6
	"log/slog"
Michael Yang's avatar
Michael Yang committed
7
	"math"
8
	"os"
9
	"reflect"
10
	"strconv"
11
	"strings"
12
	"time"
13
14

	"github.com/ollama/ollama/envconfig"
15
)
Michael Yang's avatar
Michael Yang committed
16

17
// StatusError is an error with an HTTP status code and message.
Patrick Devine's avatar
Patrick Devine committed
18
type StatusError struct {
19
20
21
	StatusCode   int
	Status       string
	ErrorMessage string `json:"error"`
Patrick Devine's avatar
Patrick Devine committed
22
23
24
}

func (e StatusError) Error() string {
25
26
27
28
29
30
31
32
33
34
	switch {
	case e.Status != "" && e.ErrorMessage != "":
		return fmt.Sprintf("%s: %s", e.Status, e.ErrorMessage)
	case e.Status != "":
		return e.Status
	case e.ErrorMessage != "":
		return e.ErrorMessage
	default:
		// this should not happen
		return "something went wrong, please see the ollama server logs for details"
Patrick Devine's avatar
Patrick Devine committed
35
36
37
	}
}

38
// ImageData represents the raw binary data of an image file.
Patrick Devine's avatar
Patrick Devine committed
39
40
type ImageData []byte

41
42
43
// GenerateRequest describes a request sent by [Client.Generate]. While you
// have to specify the Model and Prompt fields, all the other fields have
// reasonable defaults for basic uses.
44
type GenerateRequest struct {
45
46
47
	// Model is the model name; it should be a name familiar to Ollama from
	// the library at https://ollama.com/library
	Model string `json:"model"`
48

49
50
51
	// Prompt is the textual prompt to send to the model.
	Prompt string `json:"prompt"`

52
53
54
	// Suffix is the text that comes after the inserted text.
	Suffix string `json:"suffix"`

55
56
57
58
59
60
61
	// System overrides the model's default system message/prompt.
	System string `json:"system"`

	// Template overrides the model's default prompt template.
	Template string `json:"template"`

	// Context is the context parameter returned from a previous call to
62
	// [Client.Generate]. It can be used to keep a short conversational memory.
63
64
65
66
67
68
69
70
71
	Context []int `json:"context,omitempty"`

	// Stream specifies whether the response is streaming; it is true by default.
	Stream *bool `json:"stream,omitempty"`

	// Raw set to true means that no formatting will be applied to the prompt.
	Raw bool `json:"raw,omitempty"`

	// Format specifies the format to return a response in.
72
	Format json.RawMessage `json:"format,omitempty"`
73
74
75
76
77
78
79
80
81
82
83

	// KeepAlive controls how long the model will stay loaded in memory following
	// this request.
	KeepAlive *Duration `json:"keep_alive,omitempty"`

	// Images is an optional list of base64-encoded images accompanying this
	// request, for multimodal models.
	Images []ImageData `json:"images,omitempty"`

	// Options lists model-specific options. For example, temperature can be
	// set through this field, if the model supports it.
84
	Options map[string]interface{} `json:"options"`
85
86
}

87
// ChatRequest describes a request sent by [Client.Chat].
Bruce MacDonald's avatar
Bruce MacDonald committed
88
type ChatRequest struct {
89
90
91
92
93
94
	// Model is the model name, as in [GenerateRequest].
	Model string `json:"model"`

	// Messages is the messages of the chat - can be used to keep a chat memory.
	Messages []Message `json:"messages"`

95
	// Stream enables streaming of returned responses; true by default.
96
97
98
	Stream *bool `json:"stream,omitempty"`

	// Format is the format to return the response in (e.g. "json").
99
	Format json.RawMessage `json:"format,omitempty"`
100
101

	// KeepAlive controls how long the model will stay loaded into memory
102
	// following the request.
103
	KeepAlive *Duration `json:"keep_alive,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
104

Michael Yang's avatar
tools  
Michael Yang committed
105
	// Tools is an optional list of tools the model has access to.
106
	Tools `json:"tools,omitempty"`
Michael Yang's avatar
tools  
Michael Yang committed
107

108
	// Options lists model-specific options.
Bruce MacDonald's avatar
Bruce MacDonald committed
109
110
111
	Options map[string]interface{} `json:"options"`
}

112
113
114
115
116
117
118
type Tools []Tool

func (t Tools) String() string {
	bts, _ := json.Marshal(t)
	return string(bts)
}

119
120
121
122
123
func (t Tool) String() string {
	bts, _ := json.Marshal(t)
	return string(bts)
}

124
125
126
// Message is a single message in a chat sequence. The message contains the
// role ("system", "user", or "assistant"), the content and an optional list
// of images.
Bruce MacDonald's avatar
Bruce MacDonald committed
127
type Message struct {
Michael Yang's avatar
tools  
Michael Yang committed
128
	Role      string      `json:"role"`
129
	Content   string      `json:"content"`
Michael Yang's avatar
tools  
Michael Yang committed
130
131
132
133
	Images    []ImageData `json:"images,omitempty"`
	ToolCalls []ToolCall  `json:"tool_calls,omitempty"`
}

134
135
136
137
138
139
140
141
142
143
144
145
func (m *Message) UnmarshalJSON(b []byte) error {
	type Alias Message
	var a Alias
	if err := json.Unmarshal(b, &a); err != nil {
		return err
	}

	*m = Message(a)
	m.Role = strings.ToLower(m.Role)
	return nil
}

146
147
148
149
150
type ToolCall struct {
	Function ToolCallFunction `json:"function"`
}

type ToolCallFunction struct {
151
	Index     int                       `json:"index,omitempty"`
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
	Name      string                    `json:"name"`
	Arguments ToolCallFunctionArguments `json:"arguments"`
}

type ToolCallFunctionArguments map[string]any

func (t *ToolCallFunctionArguments) String() string {
	bts, _ := json.Marshal(t)
	return string(bts)
}

type Tool struct {
	Type     string       `json:"type"`
	Function ToolFunction `json:"function"`
}

type ToolFunction struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	Parameters  struct {
		Type       string   `json:"type"`
		Required   []string `json:"required"`
		Properties map[string]struct {
			Type        string   `json:"type"`
			Description string   `json:"description"`
			Enum        []string `json:"enum,omitempty"`
		} `json:"properties"`
	} `json:"parameters"`
}

func (t *ToolFunction) String() string {
	bts, _ := json.Marshal(t)
	return string(bts)
}

187
188
// ChatResponse is the response returned by [Client.Chat]. Its fields are
// similar to [GenerateResponse].
Bruce MacDonald's avatar
Bruce MacDonald committed
189
type ChatResponse struct {
190
191
192
	Model      string    `json:"model"`
	CreatedAt  time.Time `json:"created_at"`
	Message    Message   `json:"message"`
Bruce MacDonald's avatar
Bruce MacDonald committed
193
	DoneReason string    `json:"done_reason,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208

	Done bool `json:"done"`

	Metrics
}

type Metrics struct {
	TotalDuration      time.Duration `json:"total_duration,omitempty"`
	LoadDuration       time.Duration `json:"load_duration,omitempty"`
	PromptEvalCount    int           `json:"prompt_eval_count,omitempty"`
	PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
	EvalCount          int           `json:"eval_count,omitempty"`
	EvalDuration       time.Duration `json:"eval_duration,omitempty"`
}

209
210
// Options specified in [GenerateRequest].  If you add a new option here, also
// add it to the API docs.
211
212
213
214
215
216
217
218
219
type Options struct {
	Runner

	// Predict options used at runtime
	NumKeep          int      `json:"num_keep,omitempty"`
	Seed             int      `json:"seed,omitempty"`
	NumPredict       int      `json:"num_predict,omitempty"`
	TopK             int      `json:"top_k,omitempty"`
	TopP             float32  `json:"top_p,omitempty"`
220
	MinP             float32  `json:"min_p,omitempty"`
221
222
223
224
225
226
227
228
229
230
231
232
233
234
	TypicalP         float32  `json:"typical_p,omitempty"`
	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
	Temperature      float32  `json:"temperature,omitempty"`
	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
	Mirostat         int      `json:"mirostat,omitempty"`
	MirostatTau      float32  `json:"mirostat_tau,omitempty"`
	MirostatEta      float32  `json:"mirostat_eta,omitempty"`
	Stop             []string `json:"stop,omitempty"`
}

// Runner options which must be set when the model is loaded into memory
type Runner struct {
235
236
237
238
239
	NumCtx    int   `json:"num_ctx,omitempty"`
	NumBatch  int   `json:"num_batch,omitempty"`
	NumGPU    int   `json:"num_gpu,omitempty"`
	MainGPU   int   `json:"main_gpu,omitempty"`
	LowVRAM   bool  `json:"low_vram,omitempty"`
240
	F16KV     bool  `json:"f16_kv,omitempty"` // Deprecated: This option is ignored
241
242
243
244
245
	LogitsAll bool  `json:"logits_all,omitempty"`
	VocabOnly bool  `json:"vocab_only,omitempty"`
	UseMMap   *bool `json:"use_mmap,omitempty"`
	UseMLock  bool  `json:"use_mlock,omitempty"`
	NumThread int   `json:"num_thread,omitempty"`
246
247
}

248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
// EmbedRequest is the request passed to [Client.Embed].
type EmbedRequest struct {
	// Model is the model name.
	Model string `json:"model"`

	// Input is the input to embed.
	Input any `json:"input"`

	// KeepAlive controls how long the model will stay loaded in memory following
	// this request.
	KeepAlive *Duration `json:"keep_alive,omitempty"`

	Truncate *bool `json:"truncate,omitempty"`

	// Options lists model-specific options.
	Options map[string]interface{} `json:"options"`
}

// EmbedResponse is the response from [Client.Embed].
type EmbedResponse struct {
	Model      string      `json:"model"`
269
	Embeddings [][]float32 `json:"embeddings"`
270
271
272
273

	TotalDuration   time.Duration `json:"total_duration,omitempty"`
	LoadDuration    time.Duration `json:"load_duration,omitempty"`
	PromptEvalCount int           `json:"prompt_eval_count,omitempty"`
274
275
}

276
// EmbeddingRequest is the request passed to [Client.Embeddings].
Bruce MacDonald's avatar
Bruce MacDonald committed
277
type EmbeddingRequest struct {
278
279
280
281
282
283
284
285
	// Model is the model name.
	Model string `json:"model"`

	// Prompt is the textual prompt to embed.
	Prompt string `json:"prompt"`

	// KeepAlive controls how long the model will stay loaded in memory following
	// this request.
286
	KeepAlive *Duration `json:"keep_alive,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
287

288
	// Options lists model-specific options.
Bruce MacDonald's avatar
Bruce MacDonald committed
289
290
291
	Options map[string]interface{} `json:"options"`
}

292
// EmbeddingResponse is the response from [Client.Embeddings].
Bruce MacDonald's avatar
Bruce MacDonald committed
293
294
295
296
type EmbeddingResponse struct {
	Embedding []float64 `json:"embedding"`
}

297
// CreateRequest is the request passed to [Client.Create].
298
type CreateRequest struct {
299
300
301
302
303
304
305
306
307
308
309
310
311
	Model    string `json:"model"`
	Stream   *bool  `json:"stream,omitempty"`
	Quantize string `json:"quantize,omitempty"`

	From       string            `json:"from,omitempty"`
	Files      map[string]string `json:"files,omitempty"`
	Adapters   map[string]string `json:"adapters,omitempty"`
	Template   string            `json:"template,omitempty"`
	License    any               `json:"license,omitempty"`
	System     string            `json:"system,omitempty"`
	Parameters map[string]any    `json:"parameters,omitempty"`
	Messages   []Message         `json:"messages,omitempty"`

Michael Yang's avatar
Michael Yang committed
312
	// Deprecated: set the model name with Model instead
Michael Yang's avatar
Michael Yang committed
313
	Name string `json:"name"`
Michael Yang's avatar
Michael Yang committed
314
	// Deprecated: use Quantize instead
315
	Quantization string `json:"quantization,omitempty"`
316
317
}

318
// DeleteRequest is the request passed to [Client.Delete].
319
type DeleteRequest struct {
Michael Yang's avatar
Michael Yang committed
320
321
	Model string `json:"model"`

Michael Yang's avatar
Michael Yang committed
322
	// Deprecated: set the model name with Model instead
323
324
325
	Name string `json:"name"`
}

326
// ShowRequest is the request passed to [Client.Show].
Patrick Devine's avatar
Patrick Devine committed
327
type ShowRequest struct {
328
329
	Model  string `json:"model"`
	System string `json:"system"`
Patrick Devine's avatar
Patrick Devine committed
330
331

	// Template is deprecated
332
	Template string `json:"template"`
333
	Verbose  bool   `json:"verbose"`
334
335

	Options map[string]interface{} `json:"options"`
Michael Yang's avatar
Michael Yang committed
336

Michael Yang's avatar
Michael Yang committed
337
	// Deprecated: set the model name with Model instead
Michael Yang's avatar
Michael Yang committed
338
	Name string `json:"name"`
Patrick Devine's avatar
Patrick Devine committed
339
340
}

341
// ShowResponse is the response returned from [Client.Show].
Patrick Devine's avatar
Patrick Devine committed
342
type ShowResponse struct {
343
344
345
346
347
348
349
350
351
352
	License       string         `json:"license,omitempty"`
	Modelfile     string         `json:"modelfile,omitempty"`
	Parameters    string         `json:"parameters,omitempty"`
	Template      string         `json:"template,omitempty"`
	System        string         `json:"system,omitempty"`
	Details       ModelDetails   `json:"details,omitempty"`
	Messages      []Message      `json:"messages,omitempty"`
	ModelInfo     map[string]any `json:"model_info,omitempty"`
	ProjectorInfo map[string]any `json:"projector_info,omitempty"`
	ModifiedAt    time.Time      `json:"modified_at,omitempty"`
Patrick Devine's avatar
Patrick Devine committed
353
354
}

355
// CopyRequest is the request passed to [Client.Copy].
Patrick Devine's avatar
Patrick Devine committed
356
357
358
359
360
type CopyRequest struct {
	Source      string `json:"source"`
	Destination string `json:"destination"`
}

361
// PullRequest is the request passed to [Client.Pull].
Bruce MacDonald's avatar
Bruce MacDonald committed
362
type PullRequest struct {
Michael Yang's avatar
Michael Yang committed
363
	Model    string `json:"model"`
364
365
366
	Insecure bool   `json:"insecure,omitempty"` // Deprecated: ignored
	Username string `json:"username"`           // Deprecated: ignored
	Password string `json:"password"`           // Deprecated: ignored
367
	Stream   *bool  `json:"stream,omitempty"`
Michael Yang's avatar
Michael Yang committed
368

Michael Yang's avatar
Michael Yang committed
369
	// Deprecated: set the model name with Model instead
Michael Yang's avatar
Michael Yang committed
370
	Name string `json:"name"`
Bruce MacDonald's avatar
Bruce MacDonald committed
371
372
}

373
374
// ProgressResponse is the response passed to progress functions like
// [PullProgressFunc] and [PushProgressFunc].
375
type ProgressResponse struct {
376
377
	Status    string `json:"status"`
	Digest    string `json:"digest,omitempty"`
Michael Yang's avatar
Michael Yang committed
378
379
	Total     int64  `json:"total,omitempty"`
	Completed int64  `json:"completed,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
380
381
}

382
// PushRequest is the request passed to [Client.Push].
383
type PushRequest struct {
Michael Yang's avatar
Michael Yang committed
384
	Model    string `json:"model"`
385
	Insecure bool   `json:"insecure,omitempty"`
386
387
388
	Username string `json:"username"`
	Password string `json:"password"`
	Stream   *bool  `json:"stream,omitempty"`
Michael Yang's avatar
Michael Yang committed
389

Michael Yang's avatar
Michael Yang committed
390
	// Deprecated: set the model name with Model instead
Michael Yang's avatar
Michael Yang committed
391
	Name string `json:"name"`
392
}
393

394
// ListResponse is the response from [Client.List].
Patrick Devine's avatar
Patrick Devine committed
395
type ListResponse struct {
396
	Models []ListModelResponse `json:"models"`
Patrick Devine's avatar
Patrick Devine committed
397
398
}

399
400
401
402
403
404
405
// ProcessResponse is the response from [Client.Process].
type ProcessResponse struct {
	Models []ProcessModelResponse `json:"models"`
}

// ListModelResponse is a single model description in [ListResponse].
type ListModelResponse struct {
Patrick Devine's avatar
Patrick Devine committed
406
	Name       string       `json:"name"`
Michael Yang's avatar
Michael Yang committed
407
	Model      string       `json:"model"`
408
	ModifiedAt time.Time    `json:"modified_at"`
Patrick Devine's avatar
Patrick Devine committed
409
410
411
	Size       int64        `json:"size"`
	Digest     string       `json:"digest"`
	Details    ModelDetails `json:"details,omitempty"`
412
413
414
415
416
417
418
419
420
421
422
}

// ProcessModelResponse is a single model description in [ProcessResponse].
type ProcessModelResponse struct {
	Name      string       `json:"name"`
	Model     string       `json:"model"`
	Size      int64        `json:"size"`
	Digest    string       `json:"digest"`
	Details   ModelDetails `json:"details,omitempty"`
	ExpiresAt time.Time    `json:"expires_at"`
	SizeVRAM  int64        `json:"size_vram"`
Patrick Devine's avatar
Patrick Devine committed
423
424
}

425
426
427
428
429
430
431
type RetrieveModelResponse struct {
	Id      string `json:"id"`
	Object  string `json:"object"`
	Created int64  `json:"created"`
	OwnedBy string `json:"owned_by"`
}

Patrick Devine's avatar
Patrick Devine committed
432
433
434
435
type TokenResponse struct {
	Token string `json:"token"`
}

436
// GenerateResponse is the response passed into [GenerateResponseFunc].
Michael Yang's avatar
Michael Yang committed
437
type GenerateResponse struct {
438
439
440
	// Model is the model name that generated the response.
	Model string `json:"model"`

Michael Yang's avatar
Michael Yang committed
441
	// CreatedAt is the timestamp of the response.
442
443
	CreatedAt time.Time `json:"created_at"`

444
445
446
447
448
449
	// Response is the textual response itself.
	Response string `json:"response"`

	// Done specifies if the response is complete.
	Done bool `json:"done"`

450
	// DoneReason is the reason the model stopped generating text.
Bruce MacDonald's avatar
Bruce MacDonald committed
451
	DoneReason string `json:"done_reason,omitempty"`
452

453
454
	// Context is an encoding of the conversation used in this response; this
	// can be sent in the next request to keep a conversational memory.
Michael Yang's avatar
Michael Yang committed
455
	Context []int `json:"context,omitempty"`
456

Bruce MacDonald's avatar
Bruce MacDonald committed
457
	Metrics
458
459
}

460
// ModelDetails provides details about a model.
Patrick Devine's avatar
Patrick Devine committed
461
type ModelDetails struct {
462
	ParentModel       string   `json:"parent_model"`
Patrick Devine's avatar
Patrick Devine committed
463
464
465
466
467
468
469
	Format            string   `json:"format"`
	Family            string   `json:"family"`
	Families          []string `json:"families"`
	ParameterSize     string   `json:"parameter_size"`
	QuantizationLevel string   `json:"quantization_level"`
}

Bruce MacDonald's avatar
Bruce MacDonald committed
470
471
472
func (m *Metrics) Summary() {
	if m.TotalDuration > 0 {
		fmt.Fprintf(os.Stderr, "total duration:       %v\n", m.TotalDuration)
473
474
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
475
476
	if m.LoadDuration > 0 {
		fmt.Fprintf(os.Stderr, "load duration:        %v\n", m.LoadDuration)
Michael Yang's avatar
Michael Yang committed
477
478
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
479
480
	if m.PromptEvalCount > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval count:    %d token(s)\n", m.PromptEvalCount)
481
482
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
483
484
485
	if m.PromptEvalDuration > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval duration: %s\n", m.PromptEvalDuration)
		fmt.Fprintf(os.Stderr, "prompt eval rate:     %.2f tokens/s\n", float64(m.PromptEvalCount)/m.PromptEvalDuration.Seconds())
486
487
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
488
489
	if m.EvalCount > 0 {
		fmt.Fprintf(os.Stderr, "eval count:           %d token(s)\n", m.EvalCount)
490
491
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
492
493
494
	if m.EvalDuration > 0 {
		fmt.Fprintf(os.Stderr, "eval duration:        %s\n", m.EvalDuration)
		fmt.Fprintf(os.Stderr, "eval rate:            %.2f tokens/s\n", float64(m.EvalCount)/m.EvalDuration.Seconds())
495
	}
496
497
}

498
499
500
501
502
503
504
505
506
507
508
509
510
511
func (opts *Options) FromMap(m map[string]interface{}) error {
	valueOpts := reflect.ValueOf(opts).Elem() // names of the fields in the options struct
	typeOpts := reflect.TypeOf(opts).Elem()   // types of the fields in the options struct

	// build map of json struct tags to their types
	jsonOpts := make(map[string]reflect.StructField)
	for _, field := range reflect.VisibleFields(typeOpts) {
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
		if jsonTag != "" {
			jsonOpts[jsonTag] = field
		}
	}

	for key, val := range m {
512
513
		opt, ok := jsonOpts[key]
		if !ok {
514
			slog.Warn("invalid option provided", "option", key)
515
516
			continue
		}
Michael Yang's avatar
Michael Yang committed
517

518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
		field := valueOpts.FieldByName(opt.Name)
		if field.IsValid() && field.CanSet() {
			if val == nil {
				continue
			}

			switch field.Kind() {
			case reflect.Int:
				switch t := val.(type) {
				case int64:
					field.SetInt(t)
				case float64:
					// when JSON unmarshals numbers, it uses float64, not int
					field.SetInt(int64(t))
				default:
					return fmt.Errorf("option %q must be of type integer", key)
				}
			case reflect.Bool:
				val, ok := val.(bool)
				if !ok {
					return fmt.Errorf("option %q must be of type boolean", key)
				}
				field.SetBool(val)
			case reflect.Float32:
				// JSON unmarshals to float64
				val, ok := val.(float64)
				if !ok {
					return fmt.Errorf("option %q must be of type float32", key)
				}
				field.SetFloat(val)
			case reflect.String:
				val, ok := val.(string)
				if !ok {
					return fmt.Errorf("option %q must be of type string", key)
				}
				field.SetString(val)
			case reflect.Slice:
				// JSON unmarshals to []interface{}, not []string
				val, ok := val.([]interface{})
				if !ok {
					return fmt.Errorf("option %q must be of type array", key)
				}
				// convert []interface{} to []string
				slice := make([]string, len(val))
				for i, item := range val {
					str, ok := item.(string)
564
					if !ok {
565
						return fmt.Errorf("option %q must be of an array of strings", key)
566
					}
567
					slice[i] = str
568
				}
569
				field.Set(reflect.ValueOf(slice))
570
571
572
573
574
575
576
577
578
579
580
			case reflect.Pointer:
				var b bool
				if field.Type() == reflect.TypeOf(&b) {
					val, ok := val.(bool)
					if !ok {
						return fmt.Errorf("option %q must be of type boolean", key)
					}
					field.Set(reflect.ValueOf(&val))
				} else {
					return fmt.Errorf("unknown type loading config params: %v %v", field.Kind(), field.Type())
				}
581
582
			default:
				return fmt.Errorf("unknown type loading config params: %v", field.Kind())
583
584
585
			}
		}
	}
586

587
588
589
	return nil
}

590
591
// DefaultOptions is the default set of options for [GenerateRequest]; these
// values are used unless the user specifies other values explicitly.
Michael Yang's avatar
Michael Yang committed
592
593
func DefaultOptions() Options {
	return Options{
594
		// options set on request to runner
595
596
597
598
		NumPredict: -1,

		// set a minimal num_keep to avoid issues on context shifts
		NumKeep:          4,
Michael Yang's avatar
Michael Yang committed
599
600
601
602
		Temperature:      0.8,
		TopK:             40,
		TopP:             0.9,
		TypicalP:         1.0,
603
604
605
606
		RepeatLastN:      64,
		RepeatPenalty:    1.1,
		PresencePenalty:  0.0,
		FrequencyPenalty: 0.0,
Michael Yang's avatar
Michael Yang committed
607
608
609
		Mirostat:         0,
		MirostatTau:      5.0,
		MirostatEta:      0.1,
610
		Seed:             -1,
Michael Yang's avatar
Michael Yang committed
611

612
613
		Runner: Runner{
			// options set when the model is loaded
614
			NumCtx:    int(envconfig.ContextLength()),
Michael Yang's avatar
Michael Yang committed
615
616
			NumBatch:  512,
			NumGPU:    -1, // -1 here indicates that NumGPU should be set dynamically
617
			NumThread: 0,  // let the runtime decide
Michael Yang's avatar
Michael Yang committed
618
619
			LowVRAM:   false,
			UseMLock:  false,
620
			UseMMap:   nil,
621
		},
Michael Yang's avatar
Michael Yang committed
622
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
623
}
Michael Yang's avatar
Michael Yang committed
624
625
626
627
628

type Duration struct {
	time.Duration
}

629
630
631
632
633
634
635
func (d Duration) MarshalJSON() ([]byte, error) {
	if d.Duration < 0 {
		return []byte("-1"), nil
	}
	return []byte("\"" + d.Duration.String() + "\""), nil
}

Michael Yang's avatar
Michael Yang committed
636
637
638
639
640
641
642
643
644
645
646
func (d *Duration) UnmarshalJSON(b []byte) (err error) {
	var v any
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	d.Duration = 5 * time.Minute

	switch t := v.(type) {
	case float64:
		if t < 0 {
bnorick's avatar
bnorick committed
647
			d.Duration = time.Duration(math.MaxInt64)
648
		} else {
649
			d.Duration = time.Duration(int(t) * int(time.Second))
Michael Yang's avatar
Michael Yang committed
650
651
652
653
654
655
		}
	case string:
		d.Duration, err = time.ParseDuration(t)
		if err != nil {
			return err
		}
656
		if d.Duration < 0 {
bnorick's avatar
bnorick committed
657
			d.Duration = time.Duration(math.MaxInt64)
658
		}
659
660
	default:
		return fmt.Errorf("Unsupported type: '%s'", reflect.TypeOf(v))
Michael Yang's avatar
Michael Yang committed
661
662
663
664
	}

	return nil
}
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715

// FormatParams converts specified parameter options to their correct types
func FormatParams(params map[string][]string) (map[string]interface{}, error) {
	opts := Options{}
	valueOpts := reflect.ValueOf(&opts).Elem() // names of the fields in the options struct
	typeOpts := reflect.TypeOf(opts)           // types of the fields in the options struct

	// build map of json struct tags to their types
	jsonOpts := make(map[string]reflect.StructField)
	for _, field := range reflect.VisibleFields(typeOpts) {
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
		if jsonTag != "" {
			jsonOpts[jsonTag] = field
		}
	}

	out := make(map[string]interface{})
	// iterate params and set values based on json struct tags
	for key, vals := range params {
		if opt, ok := jsonOpts[key]; !ok {
			return nil, fmt.Errorf("unknown parameter '%s'", key)
		} else {
			field := valueOpts.FieldByName(opt.Name)
			if field.IsValid() && field.CanSet() {
				switch field.Kind() {
				case reflect.Float32:
					floatVal, err := strconv.ParseFloat(vals[0], 32)
					if err != nil {
						return nil, fmt.Errorf("invalid float value %s", vals)
					}

					out[key] = float32(floatVal)
				case reflect.Int:
					intVal, err := strconv.ParseInt(vals[0], 10, 64)
					if err != nil {
						return nil, fmt.Errorf("invalid int value %s", vals)
					}

					out[key] = intVal
				case reflect.Bool:
					boolVal, err := strconv.ParseBool(vals[0])
					if err != nil {
						return nil, fmt.Errorf("invalid bool value %s", vals)
					}

					out[key] = boolVal
				case reflect.String:
					out[key] = vals[0]
				case reflect.Slice:
					// TODO: only string slices are supported right now
					out[key] = vals
716
717
718
719
720
721
722
723
724
725
726
				case reflect.Pointer:
					var b bool
					if field.Type() == reflect.TypeOf(&b) {
						boolVal, err := strconv.ParseBool(vals[0])
						if err != nil {
							return nil, fmt.Errorf("invalid bool value %s", vals)
						}
						out[key] = &boolVal
					} else {
						return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
					}
727
728
729
730
731
732
733
734
735
				default:
					return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
				}
			}
		}
	}

	return out, nil
}