routes.go 25.6 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
4
	"context"
Michael Yang's avatar
Michael Yang committed
5
	"encoding/json"
6
	"errors"
7
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
8
	"io"
9
	"io/fs"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
10
11
12
	"log"
	"net"
	"net/http"
13
	"os"
14
	"os/signal"
Michael Yang's avatar
Michael Yang committed
15
	"path/filepath"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
16
	"reflect"
17
	"runtime"
Patrick Devine's avatar
Patrick Devine committed
18
	"strconv"
Michael Yang's avatar
Michael Yang committed
19
	"strings"
Michael Yang's avatar
Michael Yang committed
20
	"sync"
21
	"syscall"
22
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
23

Michael Yang's avatar
Michael Yang committed
24
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
25
26
	"github.com/gin-gonic/gin"

Jeffrey Morgan's avatar
Jeffrey Morgan committed
27
	"github.com/jmorganca/ollama/api"
28
	"github.com/jmorganca/ollama/llm"
Michael Yang's avatar
Michael Yang committed
29
	"github.com/jmorganca/ollama/parser"
Michael Yang's avatar
Michael Yang committed
30
	"github.com/jmorganca/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
31
32
)

Michael Yang's avatar
Michael Yang committed
33
34
35
36
37
38
39
40
41
42
43
44
45
46
var mode string = gin.DebugMode

func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
47
var loaded struct {
Michael Yang's avatar
Michael Yang committed
48
49
	mu sync.Mutex

50
	runner llm.LLM
Michael Yang's avatar
Michael Yang committed
51
52
53

	expireAt    time.Time
	expireTimer *time.Timer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
54

55
56
	*Model
	*api.Options
Michael Yang's avatar
Michael Yang committed
57
58
}

59
60
var defaultSessionDuration = 5 * time.Minute

Bruce MacDonald's avatar
Bruce MacDonald committed
61
// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
Bruce MacDonald's avatar
Bruce MacDonald committed
62
63
64
65
66
67
68
69
func load(c *gin.Context, modelName string, reqOpts map[string]interface{}, sessionDuration time.Duration) (*Model, error) {
	model, err := GetModel(modelName)
	if err != nil {
		return nil, err
	}

	workDir := c.GetString("workDir")

70
71
72
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		log.Printf("could not load model options: %v", err)
Bruce MacDonald's avatar
Bruce MacDonald committed
73
		return nil, err
74
75
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
76
	if err := opts.FromMap(reqOpts); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
77
		return nil, err
78
79
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
80
81
	ctx := c.Request.Context()

82
	// check if the loaded model is still running in a subprocess, in case something unexpected happened
83
84
	if loaded.runner != nil {
		if err := loaded.runner.Ping(ctx); err != nil {
85
86
			log.Print("loaded llm process not responding, closing now")
			// the subprocess is no longer running, so close it
87
88
89
90
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
91
92
93
		}
	}

94
95
96
97
98
99
100
	needLoad := loaded.runner == nil || // is there a model loaded?
		loaded.ModelPath != model.ModelPath || // has the base model changed?
		!reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
		!reflect.DeepEqual(loaded.Options.Runner, opts.Runner) // have the runner options changed?

	if needLoad {
		if loaded.runner != nil {
101
			log.Println("changing loaded model")
102
103
104
105
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
106
		}
Michael Yang's avatar
Michael Yang committed
107

Michael Yang's avatar
Michael Yang committed
108
		llmRunner, err := llm.New(workDir, model.ModelPath, model.AdapterPaths, model.ProjectorPaths, opts)
Michael Yang's avatar
Michael Yang committed
109
		if err != nil {
110
111
112
113
114
115
116
			// some older models are not compatible with newer versions of llama.cpp
			// show a generalized compatibility error until there is a better way to
			// check for model compatibility
			if strings.Contains(err.Error(), "failed to load model") {
				err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
			}

Bruce MacDonald's avatar
Bruce MacDonald committed
117
			return nil, err
Michael Yang's avatar
Michael Yang committed
118
119
		}

120
121
122
		loaded.Model = model
		loaded.runner = llmRunner
		loaded.Options = &opts
Michael Yang's avatar
Michael Yang committed
123
	}
124

Michael Yang's avatar
Michael Yang committed
125
126
127
128
	// update options for the loaded llm
	// TODO(mxyng): this isn't thread safe, but it should be fine for now
	loaded.runner.SetOptions(opts)

Jeffrey Morgan's avatar
Jeffrey Morgan committed
129
	loaded.expireAt = time.Now().Add(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
130

Jeffrey Morgan's avatar
Jeffrey Morgan committed
131
132
133
134
	if loaded.expireTimer == nil {
		loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
			loaded.mu.Lock()
			defer loaded.mu.Unlock()
Michael Yang's avatar
Michael Yang committed
135

Jeffrey Morgan's avatar
Jeffrey Morgan committed
136
			if time.Now().Before(loaded.expireAt) {
Michael Yang's avatar
Michael Yang committed
137
138
139
				return
			}

140
141
			if loaded.runner != nil {
				loaded.runner.Close()
Michael Yang's avatar
Michael Yang committed
142
143
			}

144
145
146
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
147
		})
Michael Yang's avatar
Michael Yang committed
148
	}
149

Jeffrey Morgan's avatar
Jeffrey Morgan committed
150
	loaded.expireTimer.Reset(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
151
	return model, nil
Bruce MacDonald's avatar
Bruce MacDonald committed
152
153
154
155
156
157
158
159
160
}

func GenerateHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()

	var req api.GenerateRequest
Michael Yang's avatar
Michael Yang committed
161
162
163
164
165
166
167
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
168
169
170
		return
	}

171
172
173
	// validate the request
	switch {
	case req.Model == "":
174
175
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
176
177
178
	case len(req.Format) > 0 && req.Format != "json":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
		return
179
180
181
	case req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
		return
182
183
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
184
185
	sessionDuration := defaultSessionDuration
	model, err := load(c, req.Model, req.Options, sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
186
	if err != nil {
187
		var pErr *fs.PathError
Bruce MacDonald's avatar
Bruce MacDonald committed
188
189
		switch {
		case errors.As(err, &pErr):
190
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
Bruce MacDonald's avatar
Bruce MacDonald committed
191
192
193
194
		case errors.Is(err, api.ErrInvalidOpts):
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
195
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
196
197
198
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
199
200
	// an empty request loads the model
	if req.Prompt == "" && req.Template == "" && req.System == "" {
201
202
203
204
205
		c.JSON(http.StatusOK, api.GenerateResponse{
			CreatedAt:          time.Now().UTC(),
			Model:              req.Model,
			ModelConfiguration: model.Config.ModelConfiguration,
			Done:               true})
Bruce MacDonald's avatar
Bruce MacDonald committed
206
207
208
209
210
		return
	}

	checkpointLoaded := time.Now()

Bruce MacDonald's avatar
Bruce MacDonald committed
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
	var prompt string
	switch {
	case req.Raw:
		prompt = req.Prompt
	case req.Prompt != "":
		if req.Template != "" {
			// override the default model template
			model.Template = req.Template
		}

		var rebuild strings.Builder
		if req.Context != nil {
			// TODO: context is deprecated, at some point the context logic within this conditional should be removed
			prevCtx, err := loaded.runner.Decode(c.Request.Context(), req.Context)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}

			// Remove leading spaces from prevCtx if present
			prevCtx = strings.TrimPrefix(prevCtx, " ")
			rebuild.WriteString(prevCtx)
		}
		p, err := model.Prompt(PromptVars{
			System: req.System,
			Prompt: req.Prompt,
			First:  len(req.Context) == 0,
		})
239
240
241
242
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
243
244
		rebuild.WriteString(p)
		prompt = rebuild.String()
Bruce MacDonald's avatar
Bruce MacDonald committed
245
246
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
247
	ch := make(chan any)
Bruce MacDonald's avatar
Bruce MacDonald committed
248
	var generated strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
249
250
251
	go func() {
		defer close(ch)

Bruce MacDonald's avatar
Bruce MacDonald committed
252
253
		fn := func(r llm.PredictResult) {
			// Update model expiration
Bruce MacDonald's avatar
Bruce MacDonald committed
254
255
256
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)

Bruce MacDonald's avatar
Bruce MacDonald committed
257
258
259
260
			// Build up the full response
			if _, err := generated.WriteString(r.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
261
262
			}

Bruce MacDonald's avatar
Bruce MacDonald committed
263
			resp := api.GenerateResponse{
264
265
266
267
268
				Model:              r.Model,
				ModelConfiguration: model.Config.ModelConfiguration,
				CreatedAt:          r.CreatedAt,
				Done:               r.Done,
				Response:           r.Content,
Bruce MacDonald's avatar
Bruce MacDonald committed
269
270
271
272
273
274
275
276
				Metrics: api.Metrics{
					TotalDuration:      r.TotalDuration,
					LoadDuration:       r.LoadDuration,
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
Bruce MacDonald's avatar
Bruce MacDonald committed
277
278
			}

Bruce MacDonald's avatar
Bruce MacDonald committed
279
280
281
282
283
284
285
286
287
288
			if r.Done && !req.Raw {
				embd, err := loaded.runner.Encode(c.Request.Context(), req.Prompt+generated.String())
				if err != nil {
					ch <- gin.H{"error": err.Error()}
					return
				}
				resp.Context = embd
			}

			ch <- resp
Bruce MacDonald's avatar
Bruce MacDonald committed
289
290
		}

Bruce MacDonald's avatar
Bruce MacDonald committed
291
292
293
294
295
296
297
298
299
		// Start prediction
		predictReq := llm.PredictOpts{
			Model:            model.Name,
			Prompt:           prompt,
			Format:           req.Format,
			CheckpointStart:  checkpointStart,
			CheckpointLoaded: checkpointLoaded,
		}
		if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
300
301
302
303
304
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
Bruce MacDonald's avatar
Bruce MacDonald committed
305
306
307
		// Wait for the channel to close
		var r api.GenerateResponse
		var sb strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
308
		for resp := range ch {
Bruce MacDonald's avatar
Bruce MacDonald committed
309
310
			var ok bool
			if r, ok = resp.(api.GenerateResponse); !ok {
Bruce MacDonald's avatar
Bruce MacDonald committed
311
312
313
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
Bruce MacDonald's avatar
Bruce MacDonald committed
314
			sb.WriteString(r.Response)
Bruce MacDonald's avatar
Bruce MacDonald committed
315
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
316
317
		r.Response = sb.String()
		c.JSON(http.StatusOK, r)
Bruce MacDonald's avatar
Bruce MacDonald committed
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
		return
	}

	streamResponse(c, ch)
}

func EmbeddingHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	var req api.EmbeddingRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if req.Model == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
344
345
	sessionDuration := defaultSessionDuration
	_, err = load(c, req.Model, req.Options, sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
346
	if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
347
348
349
350
351
352
353
354
355
		var pErr *fs.PathError
		switch {
		case errors.As(err, &pErr):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
		case errors.Is(err, api.ErrInvalidOpts):
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
356
357
358
		return
	}

359
	if !loaded.Options.EmbeddingOnly {
Bruce MacDonald's avatar
Bruce MacDonald committed
360
361
362
363
		c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
		return
	}

364
	embedding, err := loaded.runner.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
365
366
367
368
369
370
371
372
373
374
375
376
	if err != nil {
		log.Printf("embedding generation failed: %v", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
		return
	}

	resp := api.EmbeddingResponse{
		Embedding: embedding,
	}
	c.JSON(http.StatusOK, resp)
}

377
func PullModelHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
378
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
379
380
381
382
383
384
385
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
386
387
388
		return
	}

389
390
391
392
393
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

394
395
396
	ch := make(chan any)
	go func() {
		defer close(ch)
397
398
		fn := func(r api.ProgressResponse) {
			ch <- r
399
		}
400

401
402
403
404
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

405
406
407
408
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

		if err := PullModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
409
			ch <- gin.H{"error": err.Error()}
410
411
412
		}
	}()

413
414
415
416
417
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

418
419
420
	streamResponse(c, ch)
}

421
func PushModelHandler(c *gin.Context) {
422
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
423
424
425
426
427
428
429
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
430
431
		return
	}
Michael Yang's avatar
Michael Yang committed
432

433
434
435
436
437
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

438
439
440
	ch := make(chan any)
	go func() {
		defer close(ch)
441
442
		fn := func(r api.ProgressResponse) {
			ch <- r
443
		}
444

445
446
447
448
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
449
450
451
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

452
		if err := PushModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
453
			ch <- gin.H{"error": err.Error()}
454
455
456
		}
	}()

457
458
459
460
461
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

462
463
464
	streamResponse(c, ch)
}

465
func CreateModelHandler(c *gin.Context) {
466
	var req api.CreateRequest
Michael Yang's avatar
Michael Yang committed
467
468
469
470
471
472
473
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
474
		return
475
476
	}

Michael Yang's avatar
Michael Yang committed
477
478
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
479
480
481
		return
	}

482
483
	if err := ParseModelPath(req.Name).Validate(); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
484
485
486
		return
	}

Michael Yang's avatar
Michael Yang committed
487
488
	if req.Path == "" && req.Modelfile == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"})
Michael Yang's avatar
Michael Yang committed
489
490
		return
	}
Michael Yang's avatar
Michael Yang committed
491
492
493

	var modelfile io.Reader = strings.NewReader(req.Modelfile)
	if req.Path != "" && req.Modelfile == "" {
494
		mf, err := os.Open(req.Path)
Michael Yang's avatar
Michael Yang committed
495
496
497
498
		if err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)})
			return
		}
499
		defer mf.Close()
Michael Yang's avatar
Michael Yang committed
500

501
		modelfile = mf
Michael Yang's avatar
Michael Yang committed
502
	}
Michael Yang's avatar
Michael Yang committed
503
504
505
506
507
508
509

	commands, err := parser.Parse(modelfile)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
510
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
511
512
	go func() {
		defer close(ch)
513
514
		fn := func(resp api.ProgressResponse) {
			ch <- resp
515
516
		}

517
518
519
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

520
		if err := CreateModel(ctx, req.Name, filepath.Dir(req.Path), commands, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
521
			ch <- gin.H{"error": err.Error()}
522
		}
Michael Yang's avatar
Michael Yang committed
523
	}()
Michael Yang's avatar
Michael Yang committed
524

525
526
527
528
529
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
530
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
531
532
}

533
534
func DeleteModelHandler(c *gin.Context) {
	var req api.DeleteRequest
Michael Yang's avatar
Michael Yang committed
535
536
537
538
539
540
541
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
542
543
544
		return
	}

545
546
547
548
549
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

550
551
552
553
	if err := DeleteModel(req.Name); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
554
555
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
556
557
		return
	}
Michael Yang's avatar
Michael Yang committed
558
559
560
561
562
563
564
565
566
567
568
569

	manifestsPath, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	if err := PruneDirectory(manifestsPath); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

570
	c.JSON(http.StatusOK, nil)
571
572
}

Patrick Devine's avatar
Patrick Devine committed
573
574
func ShowModelHandler(c *gin.Context) {
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
575
576
577
578
579
580
581
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
582
583
584
		return
	}

585
586
587
588
589
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
	resp, err := GetModelInfo(req.Name)
	if err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

func GetModelInfo(name string) (*api.ShowResponse, error) {
	model, err := GetModel(name)
	if err != nil {
		return nil, err
	}

	resp := &api.ShowResponse{
		License:  strings.Join(model.License, "\n"),
		System:   model.System,
		Template: model.Template,
	}

	mf, err := ShowModelfile(model)
	if err != nil {
		return nil, err
	}

	resp.Modelfile = mf

	var params []string
	cs := 30
	for k, v := range model.Options {
		switch val := v.(type) {
		case string:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
		case int:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
		case float64:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
		case bool:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
		case []interface{}:
			for _, nv := range val {
				switch nval := nv.(type) {
				case string:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
				case int:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
				case float64:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
				case bool:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
				}
			}
		}
	}
	resp.Parameters = strings.Join(params, "\n")

	return resp, nil
}

654
func ListModelsHandler(c *gin.Context) {
655
	models := make([]api.ModelResponse, 0)
Patrick Devine's avatar
Patrick Devine committed
656
657
658
659
660
	fp, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
661
662

	walkFunc := func(path string, info os.FileInfo, _ error) error {
Patrick Devine's avatar
Patrick Devine committed
663
		if !info.IsDir() {
Michael Yang's avatar
Michael Yang committed
664
665
666
			dir, file := filepath.Split(path)
			dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
			tag := strings.Join([]string{dir, file}, ":")
667

668
			mp := ParseModelPath(tag)
Patrick Devine's avatar
Patrick Devine committed
669
			manifest, digest, err := GetManifest(mp)
Patrick Devine's avatar
Patrick Devine committed
670
			if err != nil {
671
672
				log.Printf("skipping file: %s", fp)
				return nil
Patrick Devine's avatar
Patrick Devine committed
673
			}
Michael Yang's avatar
Michael Yang committed
674
675

			models = append(models, api.ModelResponse{
Patrick Devine's avatar
Patrick Devine committed
676
677
				Name:       mp.GetShortTagname(),
				Size:       manifest.GetTotalSize(),
Patrick Devine's avatar
Patrick Devine committed
678
				Digest:     digest,
Michael Yang's avatar
Michael Yang committed
679
680
				ModifiedAt: info.ModTime(),
			})
Patrick Devine's avatar
Patrick Devine committed
681
		}
Michael Yang's avatar
Michael Yang committed
682

Patrick Devine's avatar
Patrick Devine committed
683
		return nil
Michael Yang's avatar
Michael Yang committed
684
685
686
	}

	if err := filepath.Walk(fp, walkFunc); err != nil {
Patrick Devine's avatar
Patrick Devine committed
687
688
689
690
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
691
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
692
693
}

Patrick Devine's avatar
Patrick Devine committed
694
695
func CopyModelHandler(c *gin.Context) {
	var req api.CopyRequest
Michael Yang's avatar
Michael Yang committed
696
697
698
699
700
701
702
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
703
704
705
		return
	}

706
707
708
709
710
	if req.Source == "" || req.Destination == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "source add destination are required"})
		return
	}

711
712
713
714
715
	if err := ParseModelPath(req.Destination).Validate(); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
716
717
718
719
720
721
722
723
724
725
	if err := CopyModel(req.Source, req.Destination); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}
}

Michael Yang's avatar
Michael Yang committed
726
func HeadBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
727
728
729
730
731
732
733
734
735
736
737
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); err != nil {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	}

Michael Yang's avatar
Michael Yang committed
738
	c.Status(http.StatusOK)
Michael Yang's avatar
Michael Yang committed
739
740
741
}

func CreateBlobHandler(c *gin.Context) {
742
	layer, err := NewLayer(c.Request.Body, "")
Michael Yang's avatar
Michael Yang committed
743
744
745
746
747
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

748
749
	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
Michael Yang's avatar
Michael Yang committed
750
751
752
		return
	}

753
	if _, err := layer.Commit(); err != nil {
Michael Yang's avatar
Michael Yang committed
754
755
756
757
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
758
	c.Status(http.StatusCreated)
Michael Yang's avatar
Michael Yang committed
759
760
}

Michael Yang's avatar
Michael Yang committed
761
762
763
764
765
766
767
var defaultAllowOrigins = []string{
	"localhost",
	"127.0.0.1",
	"0.0.0.0",
}

func Serve(ln net.Listener, allowOrigins []string) error {
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
	if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		// clean up unused layers and manifests
		if err := PruneLayers(); err != nil {
			return err
		}

		manifestsPath, err := GetManifestPath()
		if err != nil {
			return err
		}

		if err := PruneDirectory(manifestsPath); err != nil {
			return err
		}
	}

Michael Yang's avatar
Michael Yang committed
784
785
	config := cors.DefaultConfig()
	config.AllowWildcard = true
Michael Yang's avatar
Michael Yang committed
786
787
788
789
790
791
792
793
794
795

	config.AllowOrigins = allowOrigins
	for _, allowOrigin := range defaultAllowOrigins {
		config.AllowOrigins = append(config.AllowOrigins,
			fmt.Sprintf("http://%s", allowOrigin),
			fmt.Sprintf("https://%s", allowOrigin),
			fmt.Sprintf("http://%s:*", allowOrigin),
			fmt.Sprintf("https://%s:*", allowOrigin),
		)
	}
Michael Yang's avatar
Michael Yang committed
796

797
798
799
800
801
802
	workDir, err := os.MkdirTemp("", "ollama")
	if err != nil {
		return err
	}
	defer os.RemoveAll(workDir)

Bruce MacDonald's avatar
Bruce MacDonald committed
803
	r := gin.Default()
804
805
806
807
808
809
810
	r.Use(
		cors.New(config),
		func(c *gin.Context) {
			c.Set("workDir", workDir)
			c.Next()
		},
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
811

812
813
	r.POST("/api/pull", PullModelHandler)
	r.POST("/api/generate", GenerateHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
814
	r.POST("/api/chat", ChatHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
815
	r.POST("/api/embeddings", EmbeddingHandler)
816
817
	r.POST("/api/create", CreateModelHandler)
	r.POST("/api/push", PushModelHandler)
Patrick Devine's avatar
Patrick Devine committed
818
	r.POST("/api/copy", CopyModelHandler)
819
	r.DELETE("/api/delete", DeleteModelHandler)
Patrick Devine's avatar
Patrick Devine committed
820
	r.POST("/api/show", ShowModelHandler)
Michael Yang's avatar
Michael Yang committed
821
	r.POST("/api/blobs/:digest", CreateBlobHandler)
Michael Yang's avatar
Michael Yang committed
822
	r.HEAD("/api/blobs/:digest", HeadBlobHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
823

Michael Yang's avatar
Michael Yang committed
824
825
826
827
828
829
830
831
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

		r.Handle(method, "/api/tags", ListModelsHandler)
	}

Michael Yang's avatar
Michael Yang committed
832
	log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
833
834
835
836
	s := &http.Server{
		Handler: r,
	}

837
838
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
839
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
840
841
	go func() {
		<-signals
842
843
		if loaded.runner != nil {
			loaded.runner.Close()
844
		}
845
		os.RemoveAll(workDir)
846
847
848
		os.Exit(0)
	}()

849
850
851
	if runtime.GOOS == "linux" {
		// check compatibility to log warnings
		if _, err := llm.CheckVRAM(); err != nil {
852
			log.Printf(err.Error())
853
854
855
		}
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
856
857
	return s.Serve(ln)
}
Michael Yang's avatar
Michael Yang committed
858

859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
			if errorMsg, ok := r["error"].(string); ok {
				c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
				return
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in progress response"})
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
884
func streamResponse(c *gin.Context, ch chan any) {
885
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
886
887
888
889
890
891
892
893
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
894
			log.Printf("streamResponse: json.Marshal failed with %s", err)
Michael Yang's avatar
Michael Yang committed
895
896
897
			return false
		}

898
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
899
900
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
901
			log.Printf("streamResponse: w.Write failed with %s", err)
Michael Yang's avatar
Michael Yang committed
902
903
904
905
906
907
			return false
		}

		return true
	})
}
Bruce MacDonald's avatar
Bruce MacDonald committed
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029

func ChatHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()

	var req api.ChatRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	// validate the request
	switch {
	case req.Model == "":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	case len(req.Format) > 0 && req.Format != "json":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
		return
	}

	sessionDuration := defaultSessionDuration
	model, err := load(c, req.Model, req.Options, sessionDuration)
	if err != nil {
		var pErr *fs.PathError
		switch {
		case errors.As(err, &pErr):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
		case errors.Is(err, api.ErrInvalidOpts):
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	// an empty request loads the model
	if len(req.Messages) == 0 {
		c.JSON(http.StatusOK, api.ChatResponse{CreatedAt: time.Now().UTC(), Model: req.Model, Done: true})
		return
	}

	checkpointLoaded := time.Now()

	prompt, err := model.ChatPrompt(req.Messages)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	ch := make(chan any)

	go func() {
		defer close(ch)

		fn := func(r llm.PredictResult) {
			// Update model expiration
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)

			resp := api.ChatResponse{
				Model:     r.Model,
				CreatedAt: r.CreatedAt,
				Done:      r.Done,
				Metrics: api.Metrics{
					TotalDuration:      r.TotalDuration,
					LoadDuration:       r.LoadDuration,
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
			}

			if !r.Done {
				resp.Message = &api.Message{Role: "assistant", Content: r.Content}
			}

			ch <- resp
		}

		// Start prediction
		predictReq := llm.PredictOpts{
			Model:            model.Name,
			Prompt:           prompt,
			Format:           req.Format,
			CheckpointStart:  checkpointStart,
			CheckpointLoaded: checkpointLoaded,
		}
		if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
		// Wait for the channel to close
		var r api.ChatResponse
		var sb strings.Builder
		for resp := range ch {
			var ok bool
			if r, ok = resp.(api.ChatResponse); !ok {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
			if r.Message != nil {
				sb.WriteString(r.Message.Content)
			}
		}
		r.Message = &api.Message{Role: "assistant", Content: sb.String()}
		c.JSON(http.StatusOK, r)
		return
	}

	streamResponse(c, ch)
}