types.go 5.32 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
package api

3
4
5
6
7
8
import (
	"fmt"
	"os"
	"runtime"
	"time"
)
Michael Yang's avatar
Michael Yang committed
9

Patrick Devine's avatar
Patrick Devine committed
10
type StatusError struct {
11
12
13
	StatusCode   int
	Status       string
	ErrorMessage string `json:"error"`
Patrick Devine's avatar
Patrick Devine committed
14
15
16
}

func (e StatusError) Error() string {
17
18
19
20
21
22
23
24
25
26
	switch {
	case e.Status != "" && e.ErrorMessage != "":
		return fmt.Sprintf("%s: %s", e.Status, e.ErrorMessage)
	case e.Status != "":
		return e.Status
	case e.ErrorMessage != "":
		return e.ErrorMessage
	default:
		// this should not happen
		return "something went wrong, please see the ollama server logs for details"
Patrick Devine's avatar
Patrick Devine committed
27
28
29
	}
}

30
type GenerateRequest struct {
Michael Yang's avatar
Michael Yang committed
31
32
33
34
	SessionID int64  `json:"session_id"`
	Model     string `json:"model"`
	Prompt    string `json:"prompt"`
	Context   []int  `json:"context,omitempty"`
35
36
37
38
39
40
41
42
43

	Options `json:"options"`
}

type CreateRequest struct {
	Name string `json:"name"`
	Path string `json:"path"`
}

44
45
46
47
type DeleteRequest struct {
	Name string `json:"name"`
}

Patrick Devine's avatar
Patrick Devine committed
48
49
50
51
52
type CopyRequest struct {
	Source      string `json:"source"`
	Destination string `json:"destination"`
}

Bruce MacDonald's avatar
Bruce MacDonald committed
53
type PullRequest struct {
54
	Name     string `json:"name"`
55
	Insecure bool   `json:"insecure,omitempty"`
56
57
	Username string `json:"username"`
	Password string `json:"password"`
Bruce MacDonald's avatar
Bruce MacDonald committed
58
59
}

60
type ProgressResponse struct {
61
62
63
64
	Status    string `json:"status"`
	Digest    string `json:"digest,omitempty"`
	Total     int    `json:"total,omitempty"`
	Completed int    `json:"completed,omitempty"`
Bruce MacDonald's avatar
Bruce MacDonald committed
65
66
}

67
68
type PushRequest struct {
	Name     string `json:"name"`
69
	Insecure bool   `json:"insecure,omitempty"`
70
71
72
	Username string `json:"username"`
	Password string `json:"password"`
}
73

Patrick Devine's avatar
Patrick Devine committed
74
75
76
77
78
79
80
81
82
83
type ListResponse struct {
	Models []ListResponseModel `json:"models"`
}

type ListResponseModel struct {
	Name       string    `json:"name"`
	ModifiedAt time.Time `json:"modified_at"`
	Size       int       `json:"size"`
}

Michael Yang's avatar
Michael Yang committed
84
type GenerateResponse struct {
Michael Yang's avatar
Michael Yang committed
85
	SessionID int64     `json:"session_id"`
86
87
88
89
	Model     string    `json:"model"`
	CreatedAt time.Time `json:"created_at"`
	Response  string    `json:"response,omitempty"`

Michael Yang's avatar
Michael Yang committed
90
91
	Done    bool  `json:"done"`
	Context []int `json:"context,omitempty"`
92
93

	TotalDuration      time.Duration `json:"total_duration,omitempty"`
Michael Yang's avatar
Michael Yang committed
94
	LoadDuration       time.Duration `json:"load_duration,omitempty"`
95
96
97
98
99
100
101
102
103
104
105
	PromptEvalCount    int           `json:"prompt_eval_count,omitempty"`
	PromptEvalDuration time.Duration `json:"prompt_eval_duration,omitempty"`
	EvalCount          int           `json:"eval_count,omitempty"`
	EvalDuration       time.Duration `json:"eval_duration,omitempty"`
}

func (r *GenerateResponse) Summary() {
	if r.TotalDuration > 0 {
		fmt.Fprintf(os.Stderr, "total duration:       %v\n", r.TotalDuration)
	}

Michael Yang's avatar
Michael Yang committed
106
107
108
109
	if r.LoadDuration > 0 {
		fmt.Fprintf(os.Stderr, "load duration:        %v\n", r.LoadDuration)
	}

110
111
112
113
114
115
116
117
118
119
120
121
122
123
	if r.PromptEvalCount > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval count:    %d token(s)\n", r.PromptEvalCount)
	}

	if r.PromptEvalDuration > 0 {
		fmt.Fprintf(os.Stderr, "prompt eval duration: %s\n", r.PromptEvalDuration)
		fmt.Fprintf(os.Stderr, "prompt eval rate:     %.2f tokens/s\n", float64(r.PromptEvalCount)/r.PromptEvalDuration.Seconds())
	}

	if r.EvalCount > 0 {
		fmt.Fprintf(os.Stderr, "eval count:           %d token(s)\n", r.EvalCount)
	}

	if r.EvalDuration > 0 {
Michael Yang's avatar
Michael Yang committed
124
		fmt.Fprintf(os.Stderr, "eval duration:        %s\n", r.EvalDuration)
125
126
		fmt.Fprintf(os.Stderr, "eval rate:            %.2f tokens/s\n", float64(r.EvalCount)/r.EvalDuration.Seconds())
	}
127
128
}

Michael Yang's avatar
Michael Yang committed
129
130
type Options struct {
	Seed int `json:"seed,omitempty"`
131

Michael Yang's avatar
Michael Yang committed
132
133
134
135
136
	// Backend options
	UseNUMA bool `json:"numa,omitempty"`

	// Model options
	NumCtx        int  `json:"num_ctx,omitempty"`
Michael Yang's avatar
Michael Yang committed
137
	NumKeep       int  `json:"num_keep,omitempty"`
Michael Yang's avatar
Michael Yang committed
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
	NumBatch      int  `json:"num_batch,omitempty"`
	NumGPU        int  `json:"num_gpu,omitempty"`
	MainGPU       int  `json:"main_gpu,omitempty"`
	LowVRAM       bool `json:"low_vram,omitempty"`
	F16KV         bool `json:"f16_kv,omitempty"`
	LogitsAll     bool `json:"logits_all,omitempty"`
	VocabOnly     bool `json:"vocab_only,omitempty"`
	UseMMap       bool `json:"use_mmap,omitempty"`
	UseMLock      bool `json:"use_mlock,omitempty"`
	EmbeddingOnly bool `json:"embedding_only,omitempty"`

	// Predict options
	RepeatLastN      int     `json:"repeat_last_n,omitempty"`
	RepeatPenalty    float32 `json:"repeat_penalty,omitempty"`
	FrequencyPenalty float32 `json:"frequency_penalty,omitempty"`
	PresencePenalty  float32 `json:"presence_penalty,omitempty"`
	Temperature      float32 `json:"temperature,omitempty"`
	TopK             int     `json:"top_k,omitempty"`
	TopP             float32 `json:"top_p,omitempty"`
	TFSZ             float32 `json:"tfs_z,omitempty"`
	TypicalP         float32 `json:"typical_p,omitempty"`
	Mirostat         int     `json:"mirostat,omitempty"`
	MirostatTau      float32 `json:"mirostat_tau,omitempty"`
	MirostatEta      float32 `json:"mirostat_eta,omitempty"`
Michael Yang's avatar
Michael Yang committed
162
	PenalizeNewline  bool    `json:"penalize_newline,omitempty"`
163

Michael Yang's avatar
Michael Yang committed
164
	NumThread int `json:"num_thread,omitempty"`
Jeffrey Morgan's avatar
Jeffrey Morgan committed
165
166
}

Michael Yang's avatar
Michael Yang committed
167
168
169
170
171
172
func DefaultOptions() Options {
	return Options{
		Seed: -1,

		UseNUMA: false,

Michael Yang's avatar
Michael Yang committed
173
		NumCtx:   2048,
Michael Yang's avatar
Michael Yang committed
174
175
176
177
178
179
180
		NumBatch: 512,
		NumGPU:   1,
		LowVRAM:  false,
		F16KV:    true,
		UseMMap:  true,
		UseMLock: false,

Michael Yang's avatar
Michael Yang committed
181
		RepeatLastN:      64,
Michael Yang's avatar
Michael Yang committed
182
183
184
185
186
187
188
189
190
191
192
		RepeatPenalty:    1.1,
		FrequencyPenalty: 0.0,
		PresencePenalty:  0.0,
		Temperature:      0.8,
		TopK:             40,
		TopP:             0.9,
		TFSZ:             1.0,
		TypicalP:         1.0,
		Mirostat:         0,
		MirostatTau:      5.0,
		MirostatEta:      0.1,
Michael Yang's avatar
Michael Yang committed
193
		PenalizeNewline:  true,
Michael Yang's avatar
Michael Yang committed
194
195
196

		NumThread: runtime.NumCPU(),
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
197
}