client.go 11.1 KB
Newer Older
1
2
// Package api implements the client-side API for code wishing to interact
// with the ollama service. The methods of the [Client] type correspond to
3
// the ollama REST API as described in [the API documentation].
4
5
// The ollama command-line client itself uses this package to interact with
// the backend service.
6
7
8
9
10
11
12
13
//
// # Examples
//
// Several examples of using this package are available [in the GitHub
// repository].
//
// [the API documentation]: https://github.com/ollama/ollama/blob/main/docs/api.md
// [in the GitHub repository]: https://github.com/ollama/ollama/tree/main/examples
Jeffrey Morgan's avatar
Jeffrey Morgan committed
14
15
16
package api

import (
Jeffrey Morgan's avatar
Jeffrey Morgan committed
17
	"bufio"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
18
19
20
	"bytes"
	"context"
	"encoding/json"
21
	"fmt"
Patrick Devine's avatar
Patrick Devine committed
22
	"io"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
23
	"net/http"
Michael Yang's avatar
Michael Yang committed
24
	"net/url"
Michael Yang's avatar
Michael Yang committed
25
26
	"runtime"

27
	"github.com/ollama/ollama/envconfig"
28
29
	"github.com/ollama/ollama/format"
	"github.com/ollama/ollama/version"
30
31
)

32
33
// Client encapsulates client state for interacting with the ollama
// service. Use [ClientFromEnvironment] to create new Clients.
Patrick Devine's avatar
Patrick Devine committed
34
type Client struct {
Michael Yang's avatar
Michael Yang committed
35
	base *url.URL
Michael Yang's avatar
Michael Yang committed
36
	http *http.Client
Michael Yang's avatar
Michael Yang committed
37
38
}

Patrick Devine's avatar
Patrick Devine committed
39
func checkError(resp *http.Response, body []byte) error {
Michael Yang's avatar
Michael Yang committed
40
	if resp.StatusCode < http.StatusBadRequest {
Patrick Devine's avatar
Patrick Devine committed
41
		return nil
Michael Yang's avatar
Michael Yang committed
42
43
	}

Patrick Devine's avatar
Patrick Devine committed
44
	apiError := StatusError{StatusCode: resp.StatusCode}
Michael Yang's avatar
Michael Yang committed
45

Patrick Devine's avatar
Patrick Devine committed
46
47
48
	err := json.Unmarshal(body, &apiError)
	if err != nil {
		// Use the full body as the message if we fail to decode a response.
49
		apiError.ErrorMessage = string(body)
Patrick Devine's avatar
Patrick Devine committed
50
51
52
	}

	return apiError
Michael Yang's avatar
Michael Yang committed
53
54
}

55
56
57
58
59
60
61
62
63
// ClientFromEnvironment creates a new [Client] using configuration from the
// environment variable OLLAMA_HOST, which points to the network host and
// port on which the ollama service is listenting. The format of this variable
// is:
//
//	<scheme>://<host>:<port>
//
// If the variable is not specified, a default ollama host and port will be
// used.
Michael Yang's avatar
Michael Yang committed
64
func ClientFromEnvironment() (*Client, error) {
65
	return &Client{
Michael Yang's avatar
host  
Michael Yang committed
66
		base: envconfig.Host(),
67
68
69
70
		http: http.DefaultClient,
	}, nil
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
71
72
73
74
75
76
77
func NewClient(base *url.URL, http *http.Client) *Client {
	return &Client{
		base: base,
		http: http,
	}
}

Patrick Devine's avatar
Patrick Devine committed
78
79
80
81
func (c *Client) do(ctx context.Context, method, path string, reqData, respData any) error {
	var reqBody io.Reader
	var data []byte
	var err error
Michael Yang's avatar
Michael Yang committed
82
83
84
85
86
87
88
89

	switch reqData := reqData.(type) {
	case io.Reader:
		// reqData is already an io.Reader
		reqBody = reqData
	case nil:
		// noop
	default:
Patrick Devine's avatar
Patrick Devine committed
90
91
92
93
		data, err = json.Marshal(reqData)
		if err != nil {
			return err
		}
Michael Yang's avatar
Michael Yang committed
94

Patrick Devine's avatar
Patrick Devine committed
95
96
97
		reqBody = bytes.NewReader(data)
	}

Michael Yang's avatar
Michael Yang committed
98
	requestURL := c.base.JoinPath(path)
Michael Yang's avatar
Michael Yang committed
99
	request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), reqBody)
Patrick Devine's avatar
Patrick Devine committed
100
101
102
103
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
104
105
106
	request.Header.Set("Content-Type", "application/json")
	request.Header.Set("Accept", "application/json")
	request.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
Patrick Devine's avatar
Patrick Devine committed
107

Michael Yang's avatar
Michael Yang committed
108
	respObj, err := c.http.Do(request)
Patrick Devine's avatar
Patrick Devine committed
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
	if err != nil {
		return err
	}
	defer respObj.Body.Close()

	respBody, err := io.ReadAll(respObj.Body)
	if err != nil {
		return err
	}

	if err := checkError(respObj, respBody); err != nil {
		return err
	}

	if len(respBody) > 0 && respData != nil {
		if err := json.Unmarshal(respBody, respData); err != nil {
			return err
		}
	}
	return nil
Jeffrey Morgan's avatar
Jeffrey Morgan committed
129
130
}

Michael Yang's avatar
Michael Yang committed
131
const maxBufferSize = 512 * format.KiloByte
132

Michael Yang's avatar
Michael Yang committed
133
func (c *Client) stream(ctx context.Context, method, path string, data any, fn func([]byte) error) error {
134
135
136
137
138
139
	var buf *bytes.Buffer
	if data != nil {
		bts, err := json.Marshal(data)
		if err != nil {
			return err
		}
Michael Yang's avatar
Michael Yang committed
140

141
		buf = bytes.NewBuffer(bts)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
142
143
	}

Michael Yang's avatar
Michael Yang committed
144
	requestURL := c.base.JoinPath(path)
Michael Yang's avatar
Michael Yang committed
145
	request, err := http.NewRequestWithContext(ctx, method, requestURL.String(), buf)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
146
147
148
149
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
150
	request.Header.Set("Content-Type", "application/json")
151
	request.Header.Set("Accept", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
152
	request.Header.Set("User-Agent", fmt.Sprintf("ollama/%s (%s %s) Go/%s", version.Version, runtime.GOARCH, runtime.GOOS, runtime.Version()))
Jeffrey Morgan's avatar
Jeffrey Morgan committed
153

Michael Yang's avatar
Michael Yang committed
154
	response, err := c.http.Do(request)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
155
156
157
	if err != nil {
		return err
	}
Michael Yang's avatar
Michael Yang committed
158
	defer response.Body.Close()
Jeffrey Morgan's avatar
Jeffrey Morgan committed
159

160
	scanner := bufio.NewScanner(response.Body)
161
162
163
	// increase the buffer size to avoid running out of space
	scanBuf := make([]byte, 0, maxBufferSize)
	scanner.Buffer(scanBuf, maxBufferSize)
164
165
	for scanner.Scan() {
		var errorResponse struct {
Michael Yang's avatar
Michael Yang committed
166
			Error string `json:"error,omitempty"`
167
168
169
170
171
172
173
		}

		bts := scanner.Bytes()
		if err := json.Unmarshal(bts, &errorResponse); err != nil {
			return fmt.Errorf("unmarshal: %w", err)
		}

Michael Yang's avatar
Michael Yang committed
174
		if errorResponse.Error != "" {
175
			return fmt.Errorf(errorResponse.Error)
Michael Yang's avatar
Michael Yang committed
176
177
		}

Michael Yang's avatar
Michael Yang committed
178
		if response.StatusCode >= http.StatusBadRequest {
Michael Yang's avatar
Michael Yang committed
179
			return StatusError{
180
181
182
				StatusCode:   response.StatusCode,
				Status:       response.Status,
				ErrorMessage: errorResponse.Error,
Michael Yang's avatar
Michael Yang committed
183
			}
184
185
		}

Michael Yang's avatar
Michael Yang committed
186
		if err := fn(bts); err != nil {
187
			return err
Jeffrey Morgan's avatar
Jeffrey Morgan committed
188
189
190
		}
	}

Michael Yang's avatar
Michael Yang committed
191
192
	return nil
}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
193

194
195
196
// GenerateResponseFunc is a function that [Client.Generate] invokes every time
// a response is received from the service. If this function returns an error,
// [Client.Generate] will stop generating and return this error.
Michael Yang's avatar
Michael Yang committed
197
type GenerateResponseFunc func(GenerateResponse) error
Jeffrey Morgan's avatar
Jeffrey Morgan committed
198

199
200
201
// Generate generates a response for a given prompt. The req parameter should
// be populated with prompt details. fn is called for each response (there may
// be multiple responses, e.g. in case streaming is enabled).
Michael Yang's avatar
Michael Yang committed
202
func (c *Client) Generate(ctx context.Context, req *GenerateRequest, fn GenerateResponseFunc) error {
203
204
205
206
207
208
209
210
	return c.stream(ctx, http.MethodPost, "/api/generate", req, func(bts []byte) error {
		var resp GenerateResponse
		if err := json.Unmarshal(bts, &resp); err != nil {
			return err
		}

		return fn(resp)
	})
Jeffrey Morgan's avatar
Jeffrey Morgan committed
211
}
Bruce MacDonald's avatar
Bruce MacDonald committed
212

213
214
215
// ChatResponseFunc is a function that [Client.Chat] invokes every time
// a response is received from the service. If this function returns an error,
// [Client.Chat] will stop generating and return this error.
Bruce MacDonald's avatar
Bruce MacDonald committed
216
217
type ChatResponseFunc func(ChatResponse) error

218
219
220
221
// Chat generates the next message in a chat. [ChatRequest] may contain a
// sequence of messages which can be used to maintain chat history with a model.
// fn is called for each response (there may be multiple responses, e.g. if case
// streaming is enabled).
Bruce MacDonald's avatar
Bruce MacDonald committed
222
223
224
225
226
227
228
229
230
231
232
func (c *Client) Chat(ctx context.Context, req *ChatRequest, fn ChatResponseFunc) error {
	return c.stream(ctx, http.MethodPost, "/api/chat", req, func(bts []byte) error {
		var resp ChatResponse
		if err := json.Unmarshal(bts, &resp); err != nil {
			return err
		}

		return fn(resp)
	})
}

233
234
235
// PullProgressFunc is a function that [Client.Pull] invokes every time there
// is progress with a "pull" request sent to the service. If this function
// returns an error, [Client.Pull] will stop the process and return this error.
236
type PullProgressFunc func(ProgressResponse) error
Michael Yang's avatar
Michael Yang committed
237

238
239
240
// Pull downloads a model from the ollama library. fn is called each time
// progress is made on the request and can be used to display a progress bar,
// etc.
Michael Yang's avatar
Michael Yang committed
241
func (c *Client) Pull(ctx context.Context, req *PullRequest, fn PullProgressFunc) error {
242
	return c.stream(ctx, http.MethodPost, "/api/pull", req, func(bts []byte) error {
243
		var resp ProgressResponse
244
245
246
247
248
249
		if err := json.Unmarshal(bts, &resp); err != nil {
			return err
		}

		return fn(resp)
	})
Bruce MacDonald's avatar
Bruce MacDonald committed
250
}
251

252
253
254
// PushProgressFunc is a function that [Client.Push] invokes when progress is
// made.
// It's similar to other progress function types like [PullProgressFunc].
255
type PushProgressFunc func(ProgressResponse) error
256

257
258
259
// Push uploads a model to the model library; requires registering for ollama.ai
// and adding a public key first. fn is called each time progress is made on
// the request and can be used to display a progress bar, etc.
260
261
func (c *Client) Push(ctx context.Context, req *PushRequest, fn PushProgressFunc) error {
	return c.stream(ctx, http.MethodPost, "/api/push", req, func(bts []byte) error {
262
		var resp ProgressResponse
263
264
265
266
267
268
269
270
		if err := json.Unmarshal(bts, &resp); err != nil {
			return err
		}

		return fn(resp)
	})
}

271
272
273
// CreateProgressFunc is a function that [Client.Create] invokes when progress
// is made.
// It's similar to other progress function types like [PullProgressFunc].
274
type CreateProgressFunc func(ProgressResponse) error
275

276
277
278
279
// Create creates a model from a [Modelfile]. fn is a progress function that
// behaves similarly to other methods (see [Client.Pull]).
//
// [Modelfile]: https://github.com/ollama/ollama/blob/main/docs/modelfile.md
280
281
func (c *Client) Create(ctx context.Context, req *CreateRequest, fn CreateProgressFunc) error {
	return c.stream(ctx, http.MethodPost, "/api/create", req, func(bts []byte) error {
282
		var resp ProgressResponse
283
284
285
286
287
288
289
		if err := json.Unmarshal(bts, &resp); err != nil {
			return err
		}

		return fn(resp)
	})
}
Patrick Devine's avatar
Patrick Devine committed
290

291
// List lists models that are available locally.
Patrick Devine's avatar
Patrick Devine committed
292
293
294
func (c *Client) List(ctx context.Context) (*ListResponse, error) {
	var lr ListResponse
	if err := c.do(ctx, http.MethodGet, "/api/tags", nil, &lr); err != nil {
295
296
297
298
299
300
		return nil, err
	}
	return &lr, nil
}

// List running models.
301
302
func (c *Client) ListRunning(ctx context.Context) (*ProcessResponse, error) {
	var lr ProcessResponse
303
	if err := c.do(ctx, http.MethodGet, "/api/ps", nil, &lr); err != nil {
Patrick Devine's avatar
Patrick Devine committed
304
305
306
307
		return nil, err
	}
	return &lr, nil
}
308

309
310
// Copy copies a model - creating a model with another name from an existing
// model.
Patrick Devine's avatar
Patrick Devine committed
311
312
313
314
315
316
317
func (c *Client) Copy(ctx context.Context, req *CopyRequest) error {
	if err := c.do(ctx, http.MethodPost, "/api/copy", req, nil); err != nil {
		return err
	}
	return nil
}

318
// Delete deletes a model and its data.
319
320
321
322
323
func (c *Client) Delete(ctx context.Context, req *DeleteRequest) error {
	if err := c.do(ctx, http.MethodDelete, "/api/delete", req, nil); err != nil {
		return err
	}
	return nil
324
}
325

326
// Show obtains model information, including details, modelfile, license etc.
Patrick Devine's avatar
Patrick Devine committed
327
328
329
330
331
332
333
334
func (c *Client) Show(ctx context.Context, req *ShowRequest) (*ShowResponse, error) {
	var resp ShowResponse
	if err := c.do(ctx, http.MethodPost, "/api/show", req, &resp); err != nil {
		return nil, err
	}
	return &resp, nil
}

335
336
// Hearbeat checks if the server has started and is responsive; if yes, it
// returns nil, otherwise an error.
337
func (c *Client) Heartbeat(ctx context.Context) error {
Bruce MacDonald's avatar
Bruce MacDonald committed
338
	if err := c.do(ctx, http.MethodHead, "/", nil, nil); err != nil {
339
340
341
342
		return err
	}
	return nil
}
343

344
345
346
347
348
349
350
351
352
353
// Embed generates embeddings from a model.
func (c *Client) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) {
	var resp EmbedResponse
	if err := c.do(ctx, http.MethodPost, "/api/embed", req, &resp); err != nil {
		return nil, err
	}
	return &resp, nil
}

// Embeddings generates an embedding from a model.
Brian Murray's avatar
Brian Murray committed
354
355
356
357
358
359
360
func (c *Client) Embeddings(ctx context.Context, req *EmbeddingRequest) (*EmbeddingResponse, error) {
	var resp EmbeddingResponse
	if err := c.do(ctx, http.MethodPost, "/api/embeddings", req, &resp); err != nil {
		return nil, err
	}
	return &resp, nil
}
Michael Yang's avatar
Michael Yang committed
361

362
363
// CreateBlob creates a blob from a file on the server. digest is the
// expected SHA256 digest of the file, and r represents the file.
Michael Yang's avatar
Michael Yang committed
364
func (c *Client) CreateBlob(ctx context.Context, digest string, r io.Reader) error {
365
	return c.do(ctx, http.MethodPost, fmt.Sprintf("/api/blobs/%s", digest), r, nil)
Michael Yang's avatar
Michael Yang committed
366
}
Michael Yang's avatar
Michael Yang committed
367

368
// Version returns the Ollama server version as a string.
Michael Yang's avatar
Michael Yang committed
369
370
371
372
373
374
375
376
377
378
379
func (c *Client) Version(ctx context.Context) (string, error) {
	var version struct {
		Version string `json:"version"`
	}

	if err := c.do(ctx, http.MethodGet, "/api/version", nil, &version); err != nil {
		return "", err
	}

	return version.Version, nil
}