types.go 28.2 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
package api

3
import (
Michael Yang's avatar
Michael Yang committed
4
	"encoding/json"
5
	"fmt"
6
	"log/slog"
Michael Yang's avatar
Michael Yang committed
7
	"math"
8
	"os"
9
	"reflect"
10
	"strconv"
11
	"strings"
12
	"time"
13
14

	"github.com/ollama/ollama/envconfig"
15
	"github.com/ollama/ollama/types/model"
16
)
Michael Yang's avatar
Michael Yang committed
17

18
// StatusError is an error with an HTTP status code and message.
Patrick Devine's avatar
Patrick Devine committed
19
type StatusError struct {
20
21
22
	StatusCode   int
	Status       string
	ErrorMessage string `json:"error"`
Patrick Devine's avatar
Patrick Devine committed
23
24
25
}

func (e StatusError) Error() string {
26
27
28
29
30
31
32
33
34
35
	switch {
	case e.Status != "" && e.ErrorMessage != "":
		return fmt.Sprintf("%s: %s", e.Status, e.ErrorMessage)
	case e.Status != "":
		return e.Status
	case e.ErrorMessage != "":
		return e.ErrorMessage
	default:
		// this should not happen
		return "something went wrong, please see the ollama server logs for details"
Patrick Devine's avatar
Patrick Devine committed
36
37
38
	}
}

39
// ImageData represents the raw binary data of an image file.
Patrick Devine's avatar
Patrick Devine committed
40
41
type ImageData []byte

42
43
44
// GenerateRequest describes a request sent by [Client.Generate]. While you
// have to specify the Model and Prompt fields, all the other fields have
// reasonable defaults for basic uses.
45
type GenerateRequest struct {
46
47
48
	// Model is the model name; it should be a name familiar to Ollama from
	// the library at https://ollama.com/library
	Model string `json:"model"`
49

50
51
52
	// Prompt is the textual prompt to send to the model.
	Prompt string `json:"prompt"`

53
54
55
	// Suffix is the text that comes after the inserted text.
	Suffix string `json:"suffix"`

56
57
58
59
60
61
62
	// System overrides the model's default system message/prompt.
	System string `json:"system"`

	// Template overrides the model's default prompt template.
	Template string `json:"template"`

	// Context is the context parameter returned from a previous call to
63
	// [Client.Generate]. It can be used to keep a short conversational memory.
64
65
66
67
68
69
70
71
72
	Context []int `json:"context,omitempty"`

	// Stream specifies whether the response is streaming; it is true by default.
	Stream *bool `json:"stream,omitempty"`

	// Raw set to true means that no formatting will be applied to the prompt.
	Raw bool `json:"raw,omitempty"`

	// Format specifies the format to return a response in.
73
	Format json.RawMessage `json:"format,omitempty"`
74
75
76
77
78

	// KeepAlive controls how long the model will stay loaded in memory following
	// this request.
	KeepAlive *Duration `json:"keep_alive,omitempty"`

79
	// Images is an optional list of raw image bytes accompanying this
80
81
82
83
84
	// request, for multimodal models.
	Images []ImageData `json:"images,omitempty"`

	// Options lists model-specific options. For example, temperature can be
	// set through this field, if the model supports it.
85
	Options map[string]any `json:"options"`
86
87

	// Think controls whether thinking/reasoning models will think before
Michael Yang's avatar
Michael Yang committed
88
89
	// responding. Can be a boolean (true/false) or a string ("high", "medium", "low")
	// for supported models. Needs to be a pointer so we can distinguish between false
90
91
	// (request that thinking _not_ be used) and unset (use the old behavior
	// before this option was introduced)
Michael Yang's avatar
Michael Yang committed
92
	Think *ThinkValue `json:"think,omitempty"`
93
94
95
96

	// DebugRenderOnly is a debug option that, when set to true, returns the rendered
	// template instead of calling the model.
	DebugRenderOnly bool `json:"_debug_render_only,omitempty"`
97
98
}

99
// ChatRequest describes a request sent by [Client.Chat].
Bruce MacDonald's avatar
Bruce MacDonald committed
100
type ChatRequest struct {
101
102
103
104
105
106
	// Model is the model name, as in [GenerateRequest].
	Model string `json:"model"`

	// Messages is the messages of the chat - can be used to keep a chat memory.
	Messages []Message `json:"messages"`

107
	// Stream enables streaming of returned responses; true by default.
108
109
110
	Stream *bool `json:"stream,omitempty"`

	// Format is the format to return the response in (e.g. "json").
111
	Format json.RawMessage `json:"format,omitempty"`
112
113

	// KeepAlive controls how long the model will stay loaded into memory
114
	// following the request.
115
	KeepAlive *Duration `json:"keep_alive,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
116

Michael Yang's avatar
tools  
Michael Yang committed
117
	// Tools is an optional list of tools the model has access to.
118
	Tools `json:"tools,omitempty"`
Michael Yang's avatar
tools  
Michael Yang committed
119

120
	// Options lists model-specific options.
121
	Options map[string]any `json:"options"`
122
123

	// Think controls whether thinking/reasoning models will think before
Michael Yang's avatar
Michael Yang committed
124
125
126
	// responding. Can be a boolean (true/false) or a string ("high", "medium", "low")
	// for supported models.
	Think *ThinkValue `json:"think,omitempty"`
127
128
129
130

	// DebugRenderOnly is a debug option that, when set to true, returns the rendered
	// template instead of calling the model.
	DebugRenderOnly bool `json:"_debug_render_only,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
131
132
}

133
134
135
136
137
138
139
type Tools []Tool

func (t Tools) String() string {
	bts, _ := json.Marshal(t)
	return string(bts)
}

140
141
142
143
144
func (t Tool) String() string {
	bts, _ := json.Marshal(t)
	return string(bts)
}

145
146
147
// Message is a single message in a chat sequence. The message contains the
// role ("system", "user", or "assistant"), the content and an optional list
// of images.
Bruce MacDonald's avatar
Bruce MacDonald committed
148
type Message struct {
149
150
151
152
153
	Role    string `json:"role"`
	Content string `json:"content"`
	// Thinking contains the text that was inside thinking tags in the
	// original model output when ChatRequest.Think is enabled.
	Thinking  string      `json:"thinking,omitempty"`
Michael Yang's avatar
tools  
Michael Yang committed
154
155
	Images    []ImageData `json:"images,omitempty"`
	ToolCalls []ToolCall  `json:"tool_calls,omitempty"`
156
	ToolName  string      `json:"tool_name,omitempty"`
Michael Yang's avatar
tools  
Michael Yang committed
157
158
}

159
160
161
162
163
164
165
166
167
168
169
170
func (m *Message) UnmarshalJSON(b []byte) error {
	type Alias Message
	var a Alias
	if err := json.Unmarshal(b, &a); err != nil {
		return err
	}

	*m = Message(a)
	m.Role = strings.ToLower(m.Role)
	return nil
}

171
172
173
174
175
type ToolCall struct {
	Function ToolCallFunction `json:"function"`
}

type ToolCallFunction struct {
176
	Index     int                       `json:"index,omitempty"`
177
178
179
180
181
182
183
184
185
186
187
188
189
	Name      string                    `json:"name"`
	Arguments ToolCallFunctionArguments `json:"arguments"`
}

type ToolCallFunctionArguments map[string]any

func (t *ToolCallFunctionArguments) String() string {
	bts, _ := json.Marshal(t)
	return string(bts)
}

type Tool struct {
	Type     string       `json:"type"`
190
	Items    any          `json:"items,omitempty"`
191
192
193
	Function ToolFunction `json:"function"`
}

194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
// PropertyType can be either a string or an array of strings
type PropertyType []string

// UnmarshalJSON implements the json.Unmarshaler interface
func (pt *PropertyType) UnmarshalJSON(data []byte) error {
	// Try to unmarshal as a string first
	var s string
	if err := json.Unmarshal(data, &s); err == nil {
		*pt = []string{s}
		return nil
	}

	// If that fails, try to unmarshal as an array of strings
	var a []string
	if err := json.Unmarshal(data, &a); err != nil {
		return err
	}
	*pt = a
	return nil
}

// MarshalJSON implements the json.Marshaler interface
func (pt PropertyType) MarshalJSON() ([]byte, error) {
	if len(pt) == 1 {
		// If there's only one type, marshal as a string
		return json.Marshal(pt[0])
	}
	// Otherwise marshal as an array
	return json.Marshal([]string(pt))
}

// String returns a string representation of the PropertyType
func (pt PropertyType) String() string {
	if len(pt) == 0 {
		return ""
	}
	if len(pt) == 1 {
		return pt[0]
	}
	return fmt.Sprintf("%v", []string(pt))
}

Devon Rifkin's avatar
Devon Rifkin committed
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
type ToolProperty struct {
	AnyOf       []ToolProperty `json:"anyOf,omitempty"`
	Type        PropertyType   `json:"type"`
	Items       any            `json:"items,omitempty"`
	Description string         `json:"description"`
	Enum        []any          `json:"enum,omitempty"`
}

// ToTypeScriptType converts a ToolProperty to a TypeScript type string
func (tp ToolProperty) ToTypeScriptType() string {
	if len(tp.AnyOf) > 0 {
		var types []string
		for _, anyOf := range tp.AnyOf {
			types = append(types, anyOf.ToTypeScriptType())
		}
		return strings.Join(types, " | ")
	}

	if len(tp.Type) == 0 {
		return "any"
	}

	if len(tp.Type) == 1 {
		return mapToTypeScriptType(tp.Type[0])
	}

	var types []string
	for _, t := range tp.Type {
		types = append(types, mapToTypeScriptType(t))
	}
	return strings.Join(types, " | ")
}

// mapToTypeScriptType maps JSON Schema types to TypeScript types
func mapToTypeScriptType(jsonType string) string {
	switch jsonType {
	case "string":
		return "string"
	case "number", "integer":
		return "number"
	case "boolean":
		return "boolean"
	case "array":
		return "any[]"
	case "object":
		return "Record<string, any>"
	case "null":
		return "null"
	default:
		return "any"
	}
}

289
290
291
292
293
294
295
296
297
298
299
300
301
type ToolFunctionParameters struct {
	Type       string                  `json:"type"`
	Defs       any                     `json:"$defs,omitempty"`
	Items      any                     `json:"items,omitempty"`
	Required   []string                `json:"required"`
	Properties map[string]ToolProperty `json:"properties"`
}

func (t *ToolFunctionParameters) String() string {
	bts, _ := json.Marshal(t)
	return string(bts)
}

302
type ToolFunction struct {
303
304
305
	Name        string                 `json:"name"`
	Description string                 `json:"description"`
	Parameters  ToolFunctionParameters `json:"parameters"`
306
307
308
309
310
311
312
}

func (t *ToolFunction) String() string {
	bts, _ := json.Marshal(t)
	return string(bts)
}

313
314
// ChatResponse is the response returned by [Client.Chat]. Its fields are
// similar to [GenerateResponse].
Bruce MacDonald's avatar
Bruce MacDonald committed
315
type ChatResponse struct {
Devon Rifkin's avatar
Devon Rifkin committed
316
317
318
319
320
	Model      string     `json:"model"`
	CreatedAt  time.Time  `json:"created_at"`
	Message    Message    `json:"message"`
	DoneReason string     `json:"done_reason,omitempty"`
	DebugInfo  *DebugInfo `json:"_debug_info,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
321
322
323
324
325
326

	Done bool `json:"done"`

	Metrics
}

327
328
329
330
331
332
// DebugInfo contains debug information for template rendering
type DebugInfo struct {
	RenderedTemplate string `json:"rendered_template"`
	ImageCount       int    `json:"image_count,omitempty"`
}

Bruce MacDonald's avatar
Bruce MacDonald committed
333
334
335
336
337
338
339
340
341
type Metrics struct {
	TotalDuration      time.Duration `json:"total_duration,omitempty"`
	LoadDuration       time.Duration `json:"load_duration,omitempty"`
	PromptEvalCount    int           `json:"prompt_eval_count,omitempty"`
	PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
	EvalCount          int           `json:"eval_count,omitempty"`
	EvalDuration       time.Duration `json:"eval_duration,omitempty"`
}

342
343
// Options specified in [GenerateRequest].  If you add a new option here, also
// add it to the API docs.
344
345
346
347
348
349
350
351
352
type Options struct {
	Runner

	// Predict options used at runtime
	NumKeep          int      `json:"num_keep,omitempty"`
	Seed             int      `json:"seed,omitempty"`
	NumPredict       int      `json:"num_predict,omitempty"`
	TopK             int      `json:"top_k,omitempty"`
	TopP             float32  `json:"top_p,omitempty"`
353
	MinP             float32  `json:"min_p,omitempty"`
354
355
356
357
358
359
360
361
362
363
364
	TypicalP         float32  `json:"typical_p,omitempty"`
	RepeatLastN      int      `json:"repeat_last_n,omitempty"`
	Temperature      float32  `json:"temperature,omitempty"`
	RepeatPenalty    float32  `json:"repeat_penalty,omitempty"`
	PresencePenalty  float32  `json:"presence_penalty,omitempty"`
	FrequencyPenalty float32  `json:"frequency_penalty,omitempty"`
	Stop             []string `json:"stop,omitempty"`
}

// Runner options which must be set when the model is loaded into memory
type Runner struct {
365
366
367
368
369
370
	NumCtx    int   `json:"num_ctx,omitempty"`
	NumBatch  int   `json:"num_batch,omitempty"`
	NumGPU    int   `json:"num_gpu,omitempty"`
	MainGPU   int   `json:"main_gpu,omitempty"`
	UseMMap   *bool `json:"use_mmap,omitempty"`
	NumThread int   `json:"num_thread,omitempty"`
371
372
}

373
374
375
376
377
378
379
380
381
382
383
384
// EmbedRequest is the request passed to [Client.Embed].
type EmbedRequest struct {
	// Model is the model name.
	Model string `json:"model"`

	// Input is the input to embed.
	Input any `json:"input"`

	// KeepAlive controls how long the model will stay loaded in memory following
	// this request.
	KeepAlive *Duration `json:"keep_alive,omitempty"`

385
	// Truncate truncates the input to fit the model's max sequence length.
386
387
	Truncate *bool `json:"truncate,omitempty"`

388
389
390
	// Dimensions truncates the output embedding to the specified dimension.
	Dimensions int `json:"dimensions,omitempty"`

391
	// Options lists model-specific options.
392
	Options map[string]any `json:"options"`
393
394
395
396
397
}

// EmbedResponse is the response from [Client.Embed].
type EmbedResponse struct {
	Model      string      `json:"model"`
398
	Embeddings [][]float32 `json:"embeddings"`
399
400
401
402

	TotalDuration   time.Duration `json:"total_duration,omitempty"`
	LoadDuration    time.Duration `json:"load_duration,omitempty"`
	PromptEvalCount int           `json:"prompt_eval_count,omitempty"`
403
404
}

405
// EmbeddingRequest is the request passed to [Client.Embeddings].
Bruce MacDonald's avatar
Bruce MacDonald committed
406
type EmbeddingRequest struct {
407
408
409
410
411
412
413
414
	// Model is the model name.
	Model string `json:"model"`

	// Prompt is the textual prompt to embed.
	Prompt string `json:"prompt"`

	// KeepAlive controls how long the model will stay loaded in memory following
	// this request.
415
	KeepAlive *Duration `json:"keep_alive,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
416

417
	// Options lists model-specific options.
418
	Options map[string]any `json:"options"`
Bruce MacDonald's avatar
Bruce MacDonald committed
419
420
}

421
// EmbeddingResponse is the response from [Client.Embeddings].
Bruce MacDonald's avatar
Bruce MacDonald committed
422
423
424
425
type EmbeddingResponse struct {
	Embedding []float64 `json:"embedding"`
}

426
// CreateRequest is the request passed to [Client.Create].
427
type CreateRequest struct {
428
429
430
431
432
433
434
435
436
437
438
439
	Model    string `json:"model"`
	Stream   *bool  `json:"stream,omitempty"`
	Quantize string `json:"quantize,omitempty"`

	From       string            `json:"from,omitempty"`
	Files      map[string]string `json:"files,omitempty"`
	Adapters   map[string]string `json:"adapters,omitempty"`
	Template   string            `json:"template,omitempty"`
	License    any               `json:"license,omitempty"`
	System     string            `json:"system,omitempty"`
	Parameters map[string]any    `json:"parameters,omitempty"`
	Messages   []Message         `json:"messages,omitempty"`
Devon Rifkin's avatar
Devon Rifkin committed
440
441
	Renderer   string            `json:"renderer,omitempty"`
	Parser     string            `json:"parser,omitempty"`
442

Michael Yang's avatar
Michael Yang committed
443
	// Deprecated: set the model name with Model instead
Michael Yang's avatar
Michael Yang committed
444
	Name string `json:"name"`
Michael Yang's avatar
Michael Yang committed
445
	// Deprecated: use Quantize instead
446
	Quantization string `json:"quantization,omitempty"`
447
448
}

449
// DeleteRequest is the request passed to [Client.Delete].
450
type DeleteRequest struct {
Michael Yang's avatar
Michael Yang committed
451
452
	Model string `json:"model"`

Michael Yang's avatar
Michael Yang committed
453
	// Deprecated: set the model name with Model instead
454
455
456
	Name string `json:"name"`
}

457
// ShowRequest is the request passed to [Client.Show].
Patrick Devine's avatar
Patrick Devine committed
458
type ShowRequest struct {
459
460
	Model  string `json:"model"`
	System string `json:"system"`
Patrick Devine's avatar
Patrick Devine committed
461
462

	// Template is deprecated
463
	Template string `json:"template"`
464
	Verbose  bool   `json:"verbose"`
465

466
	Options map[string]any `json:"options"`
Michael Yang's avatar
Michael Yang committed
467

Michael Yang's avatar
Michael Yang committed
468
	// Deprecated: set the model name with Model instead
Michael Yang's avatar
Michael Yang committed
469
	Name string `json:"name"`
Patrick Devine's avatar
Patrick Devine committed
470
471
}

472
// ShowResponse is the response returned from [Client.Show].
Patrick Devine's avatar
Patrick Devine committed
473
type ShowResponse struct {
474
475
476
477
478
	License       string             `json:"license,omitempty"`
	Modelfile     string             `json:"modelfile,omitempty"`
	Parameters    string             `json:"parameters,omitempty"`
	Template      string             `json:"template,omitempty"`
	System        string             `json:"system,omitempty"`
Devon Rifkin's avatar
Devon Rifkin committed
479
480
	Renderer      string             `json:"renderer,omitempty"`
	Parser        string             `json:"parser,omitempty"`
481
482
483
484
485
486
487
	Details       ModelDetails       `json:"details,omitempty"`
	Messages      []Message          `json:"messages,omitempty"`
	ModelInfo     map[string]any     `json:"model_info,omitempty"`
	ProjectorInfo map[string]any     `json:"projector_info,omitempty"`
	Tensors       []Tensor           `json:"tensors,omitempty"`
	Capabilities  []model.Capability `json:"capabilities,omitempty"`
	ModifiedAt    time.Time          `json:"modified_at,omitempty"`
Patrick Devine's avatar
Patrick Devine committed
488
489
}

490
// CopyRequest is the request passed to [Client.Copy].
Patrick Devine's avatar
Patrick Devine committed
491
492
493
494
495
type CopyRequest struct {
	Source      string `json:"source"`
	Destination string `json:"destination"`
}

496
// PullRequest is the request passed to [Client.Pull].
Bruce MacDonald's avatar
Bruce MacDonald committed
497
type PullRequest struct {
Michael Yang's avatar
Michael Yang committed
498
	Model    string `json:"model"`
499
500
501
	Insecure bool   `json:"insecure,omitempty"` // Deprecated: ignored
	Username string `json:"username"`           // Deprecated: ignored
	Password string `json:"password"`           // Deprecated: ignored
502
	Stream   *bool  `json:"stream,omitempty"`
Michael Yang's avatar
Michael Yang committed
503

Michael Yang's avatar
Michael Yang committed
504
	// Deprecated: set the model name with Model instead
Michael Yang's avatar
Michael Yang committed
505
	Name string `json:"name"`
Bruce MacDonald's avatar
Bruce MacDonald committed
506
507
}

508
509
// ProgressResponse is the response passed to progress functions like
// [PullProgressFunc] and [PushProgressFunc].
510
type ProgressResponse struct {
511
512
	Status    string `json:"status"`
	Digest    string `json:"digest,omitempty"`
Michael Yang's avatar
Michael Yang committed
513
514
	Total     int64  `json:"total,omitempty"`
	Completed int64  `json:"completed,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
515
516
}

517
// PushRequest is the request passed to [Client.Push].
518
type PushRequest struct {
Michael Yang's avatar
Michael Yang committed
519
	Model    string `json:"model"`
520
	Insecure bool   `json:"insecure,omitempty"`
521
522
523
	Username string `json:"username"`
	Password string `json:"password"`
	Stream   *bool  `json:"stream,omitempty"`
Michael Yang's avatar
Michael Yang committed
524

Michael Yang's avatar
Michael Yang committed
525
	// Deprecated: set the model name with Model instead
Michael Yang's avatar
Michael Yang committed
526
	Name string `json:"name"`
527
}
528

529
// ListResponse is the response from [Client.List].
Patrick Devine's avatar
Patrick Devine committed
530
type ListResponse struct {
531
	Models []ListModelResponse `json:"models"`
Patrick Devine's avatar
Patrick Devine committed
532
533
}

534
535
536
537
538
539
540
// ProcessResponse is the response from [Client.Process].
type ProcessResponse struct {
	Models []ProcessModelResponse `json:"models"`
}

// ListModelResponse is a single model description in [ListResponse].
type ListModelResponse struct {
541
542
543
544
545
546
	Name       string       `json:"name"`
	Model      string       `json:"model"`
	ModifiedAt time.Time    `json:"modified_at"`
	Size       int64        `json:"size"`
	Digest     string       `json:"digest"`
	Details    ModelDetails `json:"details,omitempty"`
547
548
549
550
}

// ProcessModelResponse is a single model description in [ProcessResponse].
type ProcessModelResponse struct {
551
552
553
554
555
556
557
558
	Name          string       `json:"name"`
	Model         string       `json:"model"`
	Size          int64        `json:"size"`
	Digest        string       `json:"digest"`
	Details       ModelDetails `json:"details,omitempty"`
	ExpiresAt     time.Time    `json:"expires_at"`
	SizeVRAM      int64        `json:"size_vram"`
	ContextLength int          `json:"context_length"`
Patrick Devine's avatar
Patrick Devine committed
559
560
}

Patrick Devine's avatar
Patrick Devine committed
561
562
563
564
type TokenResponse struct {
	Token string `json:"token"`
}

565
// GenerateResponse is the response passed into [GenerateResponseFunc].
Michael Yang's avatar
Michael Yang committed
566
type GenerateResponse struct {
567
568
569
	// Model is the model name that generated the response.
	Model string `json:"model"`

Michael Yang's avatar
Michael Yang committed
570
	// CreatedAt is the timestamp of the response.
571
572
	CreatedAt time.Time `json:"created_at"`

573
574
575
	// Response is the textual response itself.
	Response string `json:"response"`

576
577
578
579
	// Thinking contains the text that was inside thinking tags in the
	// original model output when ChatRequest.Think is enabled.
	Thinking string `json:"thinking,omitempty"`

580
581
582
	// Done specifies if the response is complete.
	Done bool `json:"done"`

583
	// DoneReason is the reason the model stopped generating text.
Bruce MacDonald's avatar
Bruce MacDonald committed
584
	DoneReason string `json:"done_reason,omitempty"`
585

586
587
	// Context is an encoding of the conversation used in this response; this
	// can be sent in the next request to keep a conversational memory.
Michael Yang's avatar
Michael Yang committed
588
	Context []int `json:"context,omitempty"`
589

Bruce MacDonald's avatar
Bruce MacDonald committed
590
	Metrics
Michael Yang's avatar
Michael Yang committed
591
592

	ToolCalls []ToolCall `json:"tool_calls,omitempty"`
Devon Rifkin's avatar
Devon Rifkin committed
593
594

	DebugInfo *DebugInfo `json:"_debug_info,omitempty"`
595
596
}

597
// ModelDetails provides details about a model.
Patrick Devine's avatar
Patrick Devine committed
598
type ModelDetails struct {
599
	ParentModel       string   `json:"parent_model"`
Patrick Devine's avatar
Patrick Devine committed
600
601
602
603
604
605
606
	Format            string   `json:"format"`
	Family            string   `json:"family"`
	Families          []string `json:"families"`
	ParameterSize     string   `json:"parameter_size"`
	QuantizationLevel string   `json:"quantization_level"`
}

607
608
609
610
611
612
613
// Tensor describes the metadata for a given tensor.
type Tensor struct {
	Name  string   `json:"name"`
	Type  string   `json:"type"`
	Shape []uint64 `json:"shape"`
}

Bruce MacDonald's avatar
Bruce MacDonald committed
614
615
616
func (m *Metrics) Summary() {
	if m.TotalDuration > 0 {
		fmt.Fprintf(os.Stderr, "total duration:       %v\n", m.TotalDuration)
617
618
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
619
620
	if m.LoadDuration > 0 {
		fmt.Fprintf(os.Stderr, "load duration:        %v\n", m.LoadDuration)
Michael Yang's avatar
Michael Yang committed
621
622
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
623
624
	if m.PromptEvalCount > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval count:    %d token(s)\n", m.PromptEvalCount)
625
626
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
627
628
629
	if m.PromptEvalDuration > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval duration: %s\n", m.PromptEvalDuration)
		fmt.Fprintf(os.Stderr, "prompt eval rate:     %.2f tokens/s\n", float64(m.PromptEvalCount)/m.PromptEvalDuration.Seconds())
630
631
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
632
633
	if m.EvalCount > 0 {
		fmt.Fprintf(os.Stderr, "eval count:           %d token(s)\n", m.EvalCount)
634
635
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
636
637
638
	if m.EvalDuration > 0 {
		fmt.Fprintf(os.Stderr, "eval duration:        %s\n", m.EvalDuration)
		fmt.Fprintf(os.Stderr, "eval rate:            %.2f tokens/s\n", float64(m.EvalCount)/m.EvalDuration.Seconds())
639
	}
640
641
}

642
func (opts *Options) FromMap(m map[string]any) error {
643
644
645
646
647
648
649
650
651
652
653
654
655
	valueOpts := reflect.ValueOf(opts).Elem() // names of the fields in the options struct
	typeOpts := reflect.TypeOf(opts).Elem()   // types of the fields in the options struct

	// build map of json struct tags to their types
	jsonOpts := make(map[string]reflect.StructField)
	for _, field := range reflect.VisibleFields(typeOpts) {
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
		if jsonTag != "" {
			jsonOpts[jsonTag] = field
		}
	}

	for key, val := range m {
656
657
		opt, ok := jsonOpts[key]
		if !ok {
658
			slog.Warn("invalid option provided", "option", key)
659
660
			continue
		}
Michael Yang's avatar
Michael Yang committed
661

662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
		field := valueOpts.FieldByName(opt.Name)
		if field.IsValid() && field.CanSet() {
			if val == nil {
				continue
			}

			switch field.Kind() {
			case reflect.Int:
				switch t := val.(type) {
				case int64:
					field.SetInt(t)
				case float64:
					// when JSON unmarshals numbers, it uses float64, not int
					field.SetInt(int64(t))
				default:
					return fmt.Errorf("option %q must be of type integer", key)
				}
			case reflect.Bool:
				val, ok := val.(bool)
				if !ok {
					return fmt.Errorf("option %q must be of type boolean", key)
				}
				field.SetBool(val)
			case reflect.Float32:
				// JSON unmarshals to float64
				val, ok := val.(float64)
				if !ok {
					return fmt.Errorf("option %q must be of type float32", key)
				}
				field.SetFloat(val)
			case reflect.String:
				val, ok := val.(string)
				if !ok {
					return fmt.Errorf("option %q must be of type string", key)
				}
				field.SetString(val)
			case reflect.Slice:
699
700
				// JSON unmarshals to []any, not []string
				val, ok := val.([]any)
701
702
703
				if !ok {
					return fmt.Errorf("option %q must be of type array", key)
				}
704
				// convert []any to []string
705
706
707
				slice := make([]string, len(val))
				for i, item := range val {
					str, ok := item.(string)
708
					if !ok {
709
						return fmt.Errorf("option %q must be of an array of strings", key)
710
					}
711
					slice[i] = str
712
				}
713
				field.Set(reflect.ValueOf(slice))
714
715
716
717
718
719
720
721
722
723
724
			case reflect.Pointer:
				var b bool
				if field.Type() == reflect.TypeOf(&b) {
					val, ok := val.(bool)
					if !ok {
						return fmt.Errorf("option %q must be of type boolean", key)
					}
					field.Set(reflect.ValueOf(&val))
				} else {
					return fmt.Errorf("unknown type loading config params: %v %v", field.Kind(), field.Type())
				}
725
726
			default:
				return fmt.Errorf("unknown type loading config params: %v", field.Kind())
727
728
729
			}
		}
	}
730

731
732
733
	return nil
}

734
735
// DefaultOptions is the default set of options for [GenerateRequest]; these
// values are used unless the user specifies other values explicitly.
Michael Yang's avatar
Michael Yang committed
736
737
func DefaultOptions() Options {
	return Options{
738
		// options set on request to runner
739
740
741
742
		NumPredict: -1,

		// set a minimal num_keep to avoid issues on context shifts
		NumKeep:          4,
Michael Yang's avatar
Michael Yang committed
743
744
745
746
		Temperature:      0.8,
		TopK:             40,
		TopP:             0.9,
		TypicalP:         1.0,
747
748
749
750
751
		RepeatLastN:      64,
		RepeatPenalty:    1.1,
		PresencePenalty:  0.0,
		FrequencyPenalty: 0.0,
		Seed:             -1,
Michael Yang's avatar
Michael Yang committed
752

753
754
		Runner: Runner{
			// options set when the model is loaded
755
			NumCtx:    int(envconfig.ContextLength()),
Michael Yang's avatar
Michael Yang committed
756
757
			NumBatch:  512,
			NumGPU:    -1, // -1 here indicates that NumGPU should be set dynamically
758
			NumThread: 0,  // let the runtime decide
759
			UseMMap:   nil,
760
		},
Michael Yang's avatar
Michael Yang committed
761
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
762
}
Michael Yang's avatar
Michael Yang committed
763

Michael Yang's avatar
Michael Yang committed
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
// ThinkValue represents a value that can be a boolean or a string ("high", "medium", "low")
type ThinkValue struct {
	// Value can be a bool or string
	Value interface{}
}

// IsValid checks if the ThinkValue is valid
func (t *ThinkValue) IsValid() bool {
	if t == nil || t.Value == nil {
		return true // nil is valid (means not set)
	}

	switch v := t.Value.(type) {
	case bool:
		return true
	case string:
		return v == "high" || v == "medium" || v == "low"
	default:
		return false
	}
}

// IsBool returns true if the value is a boolean
func (t *ThinkValue) IsBool() bool {
	if t == nil || t.Value == nil {
		return false
	}
	_, ok := t.Value.(bool)
	return ok
}

// IsString returns true if the value is a string
func (t *ThinkValue) IsString() bool {
	if t == nil || t.Value == nil {
		return false
	}
	_, ok := t.Value.(string)
	return ok
}

804
805
// Bool returns the value as a bool (true if enabled in any way)
func (t *ThinkValue) Bool() bool {
Michael Yang's avatar
Michael Yang committed
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
	if t == nil || t.Value == nil {
		return false
	}

	switch v := t.Value.(type) {
	case bool:
		return v
	case string:
		// Any string value ("high", "medium", "low") means thinking is enabled
		return v == "high" || v == "medium" || v == "low"
	default:
		return false
	}
}

821
822
// String returns the value as a string
func (t *ThinkValue) String() string {
Michael Yang's avatar
Michael Yang committed
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
	if t == nil || t.Value == nil {
		return ""
	}

	switch v := t.Value.(type) {
	case string:
		return v
	case bool:
		if v {
			return "medium" // Default level when just true
		}
		return ""
	default:
		return ""
	}
}

// UnmarshalJSON implements json.Unmarshaler
func (t *ThinkValue) UnmarshalJSON(data []byte) error {
	// Try to unmarshal as bool first
	var b bool
	if err := json.Unmarshal(data, &b); err == nil {
		t.Value = b
		return nil
	}

	// Try to unmarshal as string
	var s string
	if err := json.Unmarshal(data, &s); err == nil {
		// Validate string values
		if s != "high" && s != "medium" && s != "low" {
			return fmt.Errorf("invalid think value: %q (must be \"high\", \"medium\", \"low\", true, or false)", s)
		}
		t.Value = s
		return nil
	}

	return fmt.Errorf("think must be a boolean or string (\"high\", \"medium\", \"low\")")
}

// MarshalJSON implements json.Marshaler
func (t *ThinkValue) MarshalJSON() ([]byte, error) {
	if t == nil || t.Value == nil {
		return []byte("null"), nil
	}
	return json.Marshal(t.Value)
}

Michael Yang's avatar
Michael Yang committed
871
872
873
874
type Duration struct {
	time.Duration
}

875
876
877
878
879
880
881
func (d Duration) MarshalJSON() ([]byte, error) {
	if d.Duration < 0 {
		return []byte("-1"), nil
	}
	return []byte("\"" + d.Duration.String() + "\""), nil
}

Michael Yang's avatar
Michael Yang committed
882
883
884
885
886
887
888
889
890
891
892
func (d *Duration) UnmarshalJSON(b []byte) (err error) {
	var v any
	if err := json.Unmarshal(b, &v); err != nil {
		return err
	}

	d.Duration = 5 * time.Minute

	switch t := v.(type) {
	case float64:
		if t < 0 {
bnorick's avatar
bnorick committed
893
			d.Duration = time.Duration(math.MaxInt64)
894
		} else {
Michael Yang's avatar
Michael Yang committed
895
			d.Duration = time.Duration(t * float64(time.Second))
Michael Yang's avatar
Michael Yang committed
896
897
898
899
900
901
		}
	case string:
		d.Duration, err = time.ParseDuration(t)
		if err != nil {
			return err
		}
902
		if d.Duration < 0 {
bnorick's avatar
bnorick committed
903
			d.Duration = time.Duration(math.MaxInt64)
904
		}
905
906
	default:
		return fmt.Errorf("Unsupported type: '%s'", reflect.TypeOf(v))
Michael Yang's avatar
Michael Yang committed
907
908
909
910
	}

	return nil
}
911
912

// FormatParams converts specified parameter options to their correct types
913
func FormatParams(params map[string][]string) (map[string]any, error) {
914
915
916
917
918
919
920
921
922
923
924
925
926
	opts := Options{}
	valueOpts := reflect.ValueOf(&opts).Elem() // names of the fields in the options struct
	typeOpts := reflect.TypeOf(opts)           // types of the fields in the options struct

	// build map of json struct tags to their types
	jsonOpts := make(map[string]reflect.StructField)
	for _, field := range reflect.VisibleFields(typeOpts) {
		jsonTag := strings.Split(field.Tag.Get("json"), ",")[0]
		if jsonTag != "" {
			jsonOpts[jsonTag] = field
		}
	}

927
	out := make(map[string]any)
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
	// iterate params and set values based on json struct tags
	for key, vals := range params {
		if opt, ok := jsonOpts[key]; !ok {
			return nil, fmt.Errorf("unknown parameter '%s'", key)
		} else {
			field := valueOpts.FieldByName(opt.Name)
			if field.IsValid() && field.CanSet() {
				switch field.Kind() {
				case reflect.Float32:
					floatVal, err := strconv.ParseFloat(vals[0], 32)
					if err != nil {
						return nil, fmt.Errorf("invalid float value %s", vals)
					}

					out[key] = float32(floatVal)
				case reflect.Int:
					intVal, err := strconv.ParseInt(vals[0], 10, 64)
					if err != nil {
						return nil, fmt.Errorf("invalid int value %s", vals)
					}

					out[key] = intVal
				case reflect.Bool:
					boolVal, err := strconv.ParseBool(vals[0])
					if err != nil {
						return nil, fmt.Errorf("invalid bool value %s", vals)
					}

					out[key] = boolVal
				case reflect.String:
					out[key] = vals[0]
				case reflect.Slice:
					// TODO: only string slices are supported right now
					out[key] = vals
962
963
964
965
966
967
968
969
970
971
972
				case reflect.Pointer:
					var b bool
					if field.Type() == reflect.TypeOf(&b) {
						boolVal, err := strconv.ParseBool(vals[0])
						if err != nil {
							return nil, fmt.Errorf("invalid bool value %s", vals)
						}
						out[key] = &boolVal
					} else {
						return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
					}
973
974
975
976
977
978
979
980
981
				default:
					return nil, fmt.Errorf("unknown type %s for %s", field.Kind(), key)
				}
			}
		}
	}

	return out, nil
}