routes.go 32.6 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
4
	"context"
Michael Yang's avatar
Michael Yang committed
5
	"encoding/json"
6
	"errors"
7
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
8
	"io"
9
	"io/fs"
10
	"log/slog"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
11
12
	"net"
	"net/http"
13
	"os"
14
	"os/signal"
Michael Yang's avatar
Michael Yang committed
15
	"path/filepath"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
16
	"reflect"
17
	"runtime"
Michael Yang's avatar
Michael Yang committed
18
	"strings"
Michael Yang's avatar
Michael Yang committed
19
	"sync"
20
	"syscall"
21
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
22

Michael Yang's avatar
Michael Yang committed
23
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
24
25
	"github.com/gin-gonic/gin"

Jeffrey Morgan's avatar
Jeffrey Morgan committed
26
	"github.com/jmorganca/ollama/api"
27
	"github.com/jmorganca/ollama/gpu"
28
	"github.com/jmorganca/ollama/llm"
Michael Yang's avatar
Michael Yang committed
29
	"github.com/jmorganca/ollama/parser"
Michael Yang's avatar
Michael Yang committed
30
	"github.com/jmorganca/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
31
32
)

Michael Yang's avatar
Michael Yang committed
33
34
var mode string = gin.DebugMode

35
36
37
38
type Server struct {
	WorkDir string
}

Michael Yang's avatar
Michael Yang committed
39
40
41
42
43
44
45
46
47
48
49
50
func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
51
var loaded struct {
Michael Yang's avatar
Michael Yang committed
52
53
	mu sync.Mutex

54
	runner llm.LLM
Michael Yang's avatar
Michael Yang committed
55
56
57

	expireAt    time.Time
	expireTimer *time.Timer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
58

59
60
	*Model
	*api.Options
Michael Yang's avatar
Michael Yang committed
61
62
}

63
64
var defaultSessionDuration = 5 * time.Minute

Bruce MacDonald's avatar
Bruce MacDonald committed
65
// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
66
func load(c *gin.Context, model *Model, opts api.Options, sessionDuration time.Duration) error {
Bruce MacDonald's avatar
Bruce MacDonald committed
67
68
	workDir := c.GetString("workDir")

69
70
71
72
73
74
75
	needLoad := loaded.runner == nil || // is there a model loaded?
		loaded.ModelPath != model.ModelPath || // has the base model changed?
		!reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
		!reflect.DeepEqual(loaded.Options.Runner, opts.Runner) // have the runner options changed?

	if needLoad {
		if loaded.runner != nil {
76
			slog.Info("changing loaded model")
77
78
79
80
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
81
		}
Michael Yang's avatar
Michael Yang committed
82

Michael Yang's avatar
Michael Yang committed
83
		llmRunner, err := llm.New(workDir, model.ModelPath, model.AdapterPaths, model.ProjectorPaths, opts)
Michael Yang's avatar
Michael Yang committed
84
		if err != nil {
85
86
87
			// some older models are not compatible with newer versions of llama.cpp
			// show a generalized compatibility error until there is a better way to
			// check for model compatibility
Bruce MacDonald's avatar
Bruce MacDonald committed
88
			if errors.Is(llm.ErrUnsupportedFormat, err) || strings.Contains(err.Error(), "failed to load model") {
89
90
91
				err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
			}

92
			return err
Michael Yang's avatar
Michael Yang committed
93
94
		}

95
96
97
		loaded.Model = model
		loaded.runner = llmRunner
		loaded.Options = &opts
Michael Yang's avatar
Michael Yang committed
98
	}
99

Jeffrey Morgan's avatar
Jeffrey Morgan committed
100
	loaded.expireAt = time.Now().Add(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
101

Jeffrey Morgan's avatar
Jeffrey Morgan committed
102
103
104
105
	if loaded.expireTimer == nil {
		loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
			loaded.mu.Lock()
			defer loaded.mu.Unlock()
Michael Yang's avatar
Michael Yang committed
106

Jeffrey Morgan's avatar
Jeffrey Morgan committed
107
			if time.Now().Before(loaded.expireAt) {
Michael Yang's avatar
Michael Yang committed
108
109
110
				return
			}

111
112
			if loaded.runner != nil {
				loaded.runner.Close()
Michael Yang's avatar
Michael Yang committed
113
114
			}

115
116
117
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
118
		})
Michael Yang's avatar
Michael Yang committed
119
	}
120

Jeffrey Morgan's avatar
Jeffrey Morgan committed
121
	loaded.expireTimer.Reset(sessionDuration)
122
123
124
125
126
127
128
129
130
131
132
133
134
135
	return nil
}

func modelOptions(model *Model, requestOpts map[string]interface{}) (api.Options, error) {
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		return api.Options{}, err
	}

	if err := opts.FromMap(requestOpts); err != nil {
		return api.Options{}, err
	}

	return opts, nil
Bruce MacDonald's avatar
Bruce MacDonald committed
136
137
138
139
140
141
142
143
}

func GenerateHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()
	var req api.GenerateRequest
Michael Yang's avatar
Michael Yang committed
144
	err := c.ShouldBindJSON(&req)
Patrick Devine's avatar
Patrick Devine committed
145

Michael Yang's avatar
Michael Yang committed
146
147
148
149
150
151
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
152
153
154
		return
	}

155
156
157
	// validate the request
	switch {
	case req.Model == "":
158
159
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
160
161
162
	case len(req.Format) > 0 && req.Format != "json":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
		return
163
164
165
	case req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
		return
166
167
	}

168
	model, err := GetModel(req.Model)
Bruce MacDonald's avatar
Bruce MacDonald committed
169
	if err != nil {
170
		var pErr *fs.PathError
171
		if errors.As(err, &pErr) {
172
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
173
174
175
176
177
178
179
180
181
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	opts, err := modelOptions(model, req.Options)
	if err != nil {
		if errors.Is(err, api.ErrInvalidOpts) {
Bruce MacDonald's avatar
Bruce MacDonald committed
182
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
183
			return
184
		}
185
186
187
188
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

189
190
191
192
193
194
195
	var sessionDuration time.Duration
	if req.KeepAlive == nil {
		sessionDuration = defaultSessionDuration
	} else {
		sessionDuration = req.KeepAlive.Duration
	}

196
197
	if err := load(c, model, opts, sessionDuration); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
198
199
200
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
201
202
	// an empty request loads the model
	if req.Prompt == "" && req.Template == "" && req.System == "" {
203
		c.JSON(http.StatusOK, api.GenerateResponse{
204
205
			CreatedAt: time.Now().UTC(),
			Model:     req.Model,
Michael Yang's avatar
Michael Yang committed
206
207
			Done:      true,
		})
Bruce MacDonald's avatar
Bruce MacDonald committed
208
209
210
211
212
		return
	}

	checkpointLoaded := time.Now()

Bruce MacDonald's avatar
Bruce MacDonald committed
213
	var prompt string
214
	var promptVars PromptVars
Bruce MacDonald's avatar
Bruce MacDonald committed
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
	switch {
	case req.Raw:
		prompt = req.Prompt
	case req.Prompt != "":
		if req.Template != "" {
			// override the default model template
			model.Template = req.Template
		}

		var rebuild strings.Builder
		if req.Context != nil {
			// TODO: context is deprecated, at some point the context logic within this conditional should be removed
			prevCtx, err := loaded.runner.Decode(c.Request.Context(), req.Context)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}

			// Remove leading spaces from prevCtx if present
			prevCtx = strings.TrimPrefix(prevCtx, " ")
			rebuild.WriteString(prevCtx)
		}
237
		promptVars = PromptVars{
Bruce MacDonald's avatar
Bruce MacDonald committed
238
239
240
			System: req.System,
			Prompt: req.Prompt,
			First:  len(req.Context) == 0,
241
		}
242
243
244
245
246

		if promptVars.System == "" {
			promptVars.System = model.System
		}

247
		p, err := model.PreResponsePrompt(promptVars)
248
249
250
251
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
252
253
		rebuild.WriteString(p)
		prompt = rebuild.String()
Bruce MacDonald's avatar
Bruce MacDonald committed
254
255
	}

256
257
	slog.Debug(fmt.Sprintf("prompt: %s", prompt))

Bruce MacDonald's avatar
Bruce MacDonald committed
258
	ch := make(chan any)
Bruce MacDonald's avatar
Bruce MacDonald committed
259
	var generated strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
260
261
262
	go func() {
		defer close(ch)

Bruce MacDonald's avatar
Bruce MacDonald committed
263
264
		fn := func(r llm.PredictResult) {
			// Update model expiration
Bruce MacDonald's avatar
Bruce MacDonald committed
265
266
267
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)

Bruce MacDonald's avatar
Bruce MacDonald committed
268
269
270
271
			// Build up the full response
			if _, err := generated.WriteString(r.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
272
273
			}

Bruce MacDonald's avatar
Bruce MacDonald committed
274
			resp := api.GenerateResponse{
275
				Model:     req.Model,
276
				CreatedAt: time.Now().UTC(),
277
278
				Done:      r.Done,
				Response:  r.Content,
Bruce MacDonald's avatar
Bruce MacDonald committed
279
280
281
282
283
284
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
Bruce MacDonald's avatar
Bruce MacDonald committed
285
286
			}

287
288
289
290
291
			if r.Done {
				resp.TotalDuration = time.Since(checkpointStart)
				resp.LoadDuration = checkpointLoaded.Sub(checkpointStart)

				if !req.Raw {
292
293
294
295
296
297
298
299
					// append the generated text to the history and template it if needed
					promptVars.Response = generated.String()
					result, err := model.PostResponseTemplate(promptVars)
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
					embd, err := loaded.runner.Encode(c.Request.Context(), prompt+result)
300
301
302
303
304
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
					resp.Context = embd
Bruce MacDonald's avatar
Bruce MacDonald committed
305
306
307
308
				}
			}

			ch <- resp
Bruce MacDonald's avatar
Bruce MacDonald committed
309
310
		}

Bruce MacDonald's avatar
Bruce MacDonald committed
311
312
		// Start prediction
		predictReq := llm.PredictOpts{
313
314
315
316
			Prompt:  prompt,
			Format:  req.Format,
			Images:  req.Images,
			Options: opts,
Bruce MacDonald's avatar
Bruce MacDonald committed
317
318
		}
		if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
319
320
321
322
323
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
324
325
		// Accumulate responses into the final response
		var final api.GenerateResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
326
		var sb strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
327
		for resp := range ch {
328
329
330
331
332
333
334
335
336
337
338
339
340
341
			switch r := resp.(type) {
			case api.GenerateResponse:
				sb.WriteString(r.Response)
				final = r
			case gin.H:
				if errorMsg, ok := r["error"].(string); ok {
					c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
					return
				} else {
					c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
					return
				}
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
Bruce MacDonald's avatar
Bruce MacDonald committed
342
343
344
				return
			}
		}
345
346
347

		final.Response = sb.String()
		c.JSON(http.StatusOK, final)
Bruce MacDonald's avatar
Bruce MacDonald committed
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
		return
	}

	streamResponse(c, ch)
}

func EmbeddingHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	var req api.EmbeddingRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if req.Model == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

374
	model, err := GetModel(req.Model)
Bruce MacDonald's avatar
Bruce MacDonald committed
375
	if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
376
		var pErr *fs.PathError
377
		if errors.As(err, &pErr) {
Bruce MacDonald's avatar
Bruce MacDonald committed
378
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
379
380
381
382
383
384
385
386
387
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	opts, err := modelOptions(model, req.Options)
	if err != nil {
		if errors.Is(err, api.ErrInvalidOpts) {
Bruce MacDonald's avatar
Bruce MacDonald committed
388
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
389
			return
Bruce MacDonald's avatar
Bruce MacDonald committed
390
		}
391
392
393
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
394
395
396
397
398
399
400
401

	var sessionDuration time.Duration
	if req.KeepAlive == nil {
		sessionDuration = defaultSessionDuration
	} else {
		sessionDuration = req.KeepAlive.Duration
	}

402
403
	if err := load(c, model, opts, sessionDuration); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
404
405
406
		return
	}

407
	if !loaded.Options.EmbeddingOnly {
Bruce MacDonald's avatar
Bruce MacDonald committed
408
409
410
411
		c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
		return
	}

412
	embedding, err := loaded.runner.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
413
	if err != nil {
414
		slog.Info(fmt.Sprintf("embedding generation failed: %v", err))
Bruce MacDonald's avatar
Bruce MacDonald committed
415
416
417
418
419
420
421
422
423
424
		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
		return
	}

	resp := api.EmbeddingResponse{
		Embedding: embedding,
	}
	c.JSON(http.StatusOK, resp)
}

425
func PullModelHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
426
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
427
428
429
430
431
432
433
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
434
435
436
		return
	}

Michael Yang's avatar
Michael Yang committed
437
438
439
440
441
442
443
	var model string
	if req.Model != "" {
		model = req.Model
	} else if req.Name != "" {
		model = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
444
445
446
		return
	}

447
448
449
	ch := make(chan any)
	go func() {
		defer close(ch)
450
451
		fn := func(r api.ProgressResponse) {
			ch <- r
452
		}
453

454
455
456
457
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

458
459
460
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

Michael Yang's avatar
Michael Yang committed
461
		if err := PullModel(ctx, model, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
462
			ch <- gin.H{"error": err.Error()}
463
464
465
		}
	}()

466
467
468
469
470
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

471
472
473
	streamResponse(c, ch)
}

474
func PushModelHandler(c *gin.Context) {
475
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
476
477
478
479
480
481
482
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
483
484
		return
	}
Michael Yang's avatar
Michael Yang committed
485

Michael Yang's avatar
Michael Yang committed
486
487
488
489
490
491
492
	var model string
	if req.Model != "" {
		model = req.Model
	} else if req.Name != "" {
		model = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
493
494
495
		return
	}

496
497
498
	ch := make(chan any)
	go func() {
		defer close(ch)
499
500
		fn := func(r api.ProgressResponse) {
			ch <- r
501
		}
502

503
504
505
506
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
507
508
509
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

Michael Yang's avatar
Michael Yang committed
510
		if err := PushModel(ctx, model, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
511
			ch <- gin.H{"error": err.Error()}
512
513
514
		}
	}()

515
516
517
518
519
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

520
521
522
	streamResponse(c, ch)
}

523
func CreateModelHandler(c *gin.Context) {
524
	var req api.CreateRequest
Michael Yang's avatar
Michael Yang committed
525
526
527
528
529
530
531
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
532
		return
533
534
	}

Michael Yang's avatar
Michael Yang committed
535
536
537
538
539
540
541
	var model string
	if req.Model != "" {
		model = req.Model
	} else if req.Name != "" {
		model = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
542
543
544
		return
	}

Michael Yang's avatar
Michael Yang committed
545
	if err := ParseModelPath(model).Validate(); err != nil {
546
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
547
548
549
		return
	}

Michael Yang's avatar
Michael Yang committed
550
551
	if req.Path == "" && req.Modelfile == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"})
Michael Yang's avatar
Michael Yang committed
552
553
		return
	}
Michael Yang's avatar
Michael Yang committed
554
555
556

	var modelfile io.Reader = strings.NewReader(req.Modelfile)
	if req.Path != "" && req.Modelfile == "" {
557
		mf, err := os.Open(req.Path)
Michael Yang's avatar
Michael Yang committed
558
559
560
561
		if err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)})
			return
		}
562
		defer mf.Close()
Michael Yang's avatar
Michael Yang committed
563

564
		modelfile = mf
Michael Yang's avatar
Michael Yang committed
565
	}
Michael Yang's avatar
Michael Yang committed
566
567
568
569
570
571
572

	commands, err := parser.Parse(modelfile)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
573
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
574
575
	go func() {
		defer close(ch)
576
577
		fn := func(resp api.ProgressResponse) {
			ch <- resp
578
579
		}

580
581
582
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

Michael Yang's avatar
Michael Yang committed
583
		if err := CreateModel(ctx, model, filepath.Dir(req.Path), commands, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
584
			ch <- gin.H{"error": err.Error()}
585
		}
Michael Yang's avatar
Michael Yang committed
586
	}()
Michael Yang's avatar
Michael Yang committed
587

588
589
590
591
592
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
593
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
594
595
}

596
597
func DeleteModelHandler(c *gin.Context) {
	var req api.DeleteRequest
Michael Yang's avatar
Michael Yang committed
598
599
600
601
602
603
604
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
605
606
607
		return
	}

Michael Yang's avatar
Michael Yang committed
608
609
610
611
612
613
614
	var model string
	if req.Model != "" {
		model = req.Model
	} else if req.Name != "" {
		model = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
615
616
617
		return
	}

Michael Yang's avatar
Michael Yang committed
618
	if err := DeleteModel(model); err != nil {
619
		if os.IsNotExist(err) {
Michael Yang's avatar
Michael Yang committed
620
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", model)})
621
		} else {
622
623
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
624
625
		return
	}
Michael Yang's avatar
Michael Yang committed
626
627
628
629
630
631
632
633
634
635
636
637

	manifestsPath, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	if err := PruneDirectory(manifestsPath); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

638
	c.JSON(http.StatusOK, nil)
639
640
}

Patrick Devine's avatar
Patrick Devine committed
641
642
func ShowModelHandler(c *gin.Context) {
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
643
644
645
646
647
648
649
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
650
651
652
		return
	}

Michael Yang's avatar
Michael Yang committed
653
	if req.Model != "" {
Michael Yang's avatar
Michael Yang committed
654
		// noop
Michael Yang's avatar
Michael Yang committed
655
	} else if req.Name != "" {
Michael Yang's avatar
Michael Yang committed
656
		req.Model = req.Name
Michael Yang's avatar
Michael Yang committed
657
	} else {
658
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
659
660
661
		return
	}

662
	resp, err := GetModelInfo(req)
Patrick Devine's avatar
Patrick Devine committed
663
664
	if err != nil {
		if os.IsNotExist(err) {
Michael Yang's avatar
Michael Yang committed
665
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
Patrick Devine's avatar
Patrick Devine committed
666
667
668
669
670
671
672
673
674
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

675
676
func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
	model, err := GetModel(req.Model)
Patrick Devine's avatar
Patrick Devine committed
677
678
679
680
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
681
	modelDetails := api.ModelDetails{
682
		ParentModel:       model.ParentModel,
Patrick Devine's avatar
Patrick Devine committed
683
684
685
686
687
688
689
		Format:            model.Config.ModelFormat,
		Family:            model.Config.ModelFamily,
		Families:          model.Config.ModelFamilies,
		ParameterSize:     model.Config.ModelType,
		QuantizationLevel: model.Config.FileType,
	}

690
691
692
693
694
695
696
697
	if req.System != "" {
		model.System = req.System
	}

	if req.Template != "" {
		model.Template = req.Template
	}

698
699
700
701
702
	msgs := make([]api.Message, 0)
	for _, msg := range model.Messages {
		msgs = append(msgs, api.Message{Role: msg.Role, Content: msg.Content})
	}

Patrick Devine's avatar
Patrick Devine committed
703
704
705
706
	resp := &api.ShowResponse{
		License:  strings.Join(model.License, "\n"),
		System:   model.System,
		Template: model.Template,
Patrick Devine's avatar
Patrick Devine committed
707
		Details:  modelDetails,
708
		Messages: msgs,
Patrick Devine's avatar
Patrick Devine committed
709
710
711
712
713
714
715
716
	}

	var params []string
	cs := 30
	for k, v := range model.Options {
		switch val := v.(type) {
		case []interface{}:
			for _, nv := range val {
Patrick Devine's avatar
Patrick Devine committed
717
				params = append(params, fmt.Sprintf("%-*s %#v", cs, k, nv))
Patrick Devine's avatar
Patrick Devine committed
718
			}
Patrick Devine's avatar
Patrick Devine committed
719
720
		default:
			params = append(params, fmt.Sprintf("%-*s %#v", cs, k, v))
Patrick Devine's avatar
Patrick Devine committed
721
722
723
724
		}
	}
	resp.Parameters = strings.Join(params, "\n")

725
726
727
728
729
730
731
732
733
734
735
736
737
	for k, v := range req.Options {
		if _, ok := req.Options[k]; ok {
			model.Options[k] = v
		}
	}

	mf, err := ShowModelfile(model)
	if err != nil {
		return nil, err
	}

	resp.Modelfile = mf

Patrick Devine's avatar
Patrick Devine committed
738
739
740
	return resp, nil
}

741
func ListModelsHandler(c *gin.Context) {
742
	models := make([]api.ModelResponse, 0)
743
	manifestsPath, err := GetManifestPath()
Patrick Devine's avatar
Patrick Devine committed
744
745
746
747
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
748

Patrick Devine's avatar
Patrick Devine committed
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
	modelResponse := func(modelName string) (api.ModelResponse, error) {
		model, err := GetModel(modelName)
		if err != nil {
			return api.ModelResponse{}, err
		}

		modelDetails := api.ModelDetails{
			Format:            model.Config.ModelFormat,
			Family:            model.Config.ModelFamily,
			Families:          model.Config.ModelFamilies,
			ParameterSize:     model.Config.ModelType,
			QuantizationLevel: model.Config.FileType,
		}

		return api.ModelResponse{
Michael Yang's avatar
Michael Yang committed
764
			Model:   model.ShortName,
Patrick Devine's avatar
Patrick Devine committed
765
766
767
768
769
770
771
			Name:    model.ShortName,
			Size:    model.Size,
			Digest:  model.Digest,
			Details: modelDetails,
		}, nil
	}

Michael Yang's avatar
Michael Yang committed
772
	walkFunc := func(path string, info os.FileInfo, _ error) error {
Patrick Devine's avatar
Patrick Devine committed
773
		if !info.IsDir() {
774
775
776
777
			path, tag := filepath.Split(path)
			model := strings.Trim(strings.TrimPrefix(path, manifestsPath), string(os.PathSeparator))
			modelPath := strings.Join([]string{model, tag}, ":")
			canonicalModelPath := strings.ReplaceAll(modelPath, string(os.PathSeparator), "/")
778

779
			resp, err := modelResponse(canonicalModelPath)
Patrick Devine's avatar
Patrick Devine committed
780
			if err != nil {
781
				slog.Info(fmt.Sprintf("skipping file: %s", canonicalModelPath))
Michael Yang's avatar
Michael Yang committed
782
				// nolint: nilerr
783
				return nil
Patrick Devine's avatar
Patrick Devine committed
784
			}
Michael Yang's avatar
Michael Yang committed
785

Patrick Devine's avatar
Patrick Devine committed
786
787
			resp.ModifiedAt = info.ModTime()
			models = append(models, resp)
Patrick Devine's avatar
Patrick Devine committed
788
		}
Michael Yang's avatar
Michael Yang committed
789

Patrick Devine's avatar
Patrick Devine committed
790
		return nil
Michael Yang's avatar
Michael Yang committed
791
792
	}

793
	if err := filepath.Walk(manifestsPath, walkFunc); err != nil {
Patrick Devine's avatar
Patrick Devine committed
794
795
796
797
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
798
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
799
800
}

Patrick Devine's avatar
Patrick Devine committed
801
802
func CopyModelHandler(c *gin.Context) {
	var req api.CopyRequest
Michael Yang's avatar
Michael Yang committed
803
804
805
806
807
808
809
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
810
811
812
		return
	}

813
814
815
816
817
	if req.Source == "" || req.Destination == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "source add destination are required"})
		return
	}

818
819
820
821
822
	if err := ParseModelPath(req.Destination).Validate(); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
823
824
825
826
827
828
829
830
831
832
	if err := CopyModel(req.Source, req.Destination); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}
}

Michael Yang's avatar
Michael Yang committed
833
func HeadBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
834
835
836
837
838
839
840
841
842
843
844
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); err != nil {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	}

Michael Yang's avatar
Michael Yang committed
845
	c.Status(http.StatusOK)
Michael Yang's avatar
Michael Yang committed
846
847
848
}

func CreateBlobHandler(c *gin.Context) {
849
	layer, err := NewLayer(c.Request.Body, "")
Michael Yang's avatar
Michael Yang committed
850
851
852
853
854
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

855
856
	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
Michael Yang's avatar
Michael Yang committed
857
858
859
		return
	}

860
	if _, err := layer.Commit(); err != nil {
Michael Yang's avatar
Michael Yang committed
861
862
863
864
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
865
	c.Status(http.StatusCreated)
Michael Yang's avatar
Michael Yang committed
866
867
}

Michael Yang's avatar
Michael Yang committed
868
869
870
871
872
873
var defaultAllowOrigins = []string{
	"localhost",
	"127.0.0.1",
	"0.0.0.0",
}

874
875
876
877
878
func NewServer() (*Server, error) {
	workDir, err := os.MkdirTemp("", "ollama")
	if err != nil {
		return nil, err
	}
879

880
881
882
883
	return &Server{
		WorkDir: workDir,
	}, nil
}
884

885
886
887
888
func (s *Server) GenerateRoutes() http.Handler {
	var origins []string
	if o := os.Getenv("OLLAMA_ORIGINS"); o != "" {
		origins = strings.Split(o, ",")
889
890
	}

Michael Yang's avatar
Michael Yang committed
891
892
	config := cors.DefaultConfig()
	config.AllowWildcard = true
893
	config.AllowBrowserExtensions = true
Michael Yang's avatar
Michael Yang committed
894

895
	config.AllowOrigins = origins
Michael Yang's avatar
Michael Yang committed
896
897
898
899
900
901
902
903
	for _, allowOrigin := range defaultAllowOrigins {
		config.AllowOrigins = append(config.AllowOrigins,
			fmt.Sprintf("http://%s", allowOrigin),
			fmt.Sprintf("https://%s", allowOrigin),
			fmt.Sprintf("http://%s:*", allowOrigin),
			fmt.Sprintf("https://%s:*", allowOrigin),
		)
	}
Michael Yang's avatar
Michael Yang committed
904

Bruce MacDonald's avatar
Bruce MacDonald committed
905
	r := gin.Default()
906
907
908
	r.Use(
		cors.New(config),
		func(c *gin.Context) {
909
			c.Set("workDir", s.WorkDir)
910
911
912
			c.Next()
		},
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
913

914
915
	r.POST("/api/pull", PullModelHandler)
	r.POST("/api/generate", GenerateHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
916
	r.POST("/api/chat", ChatHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
917
	r.POST("/api/embeddings", EmbeddingHandler)
918
919
	r.POST("/api/create", CreateModelHandler)
	r.POST("/api/push", PushModelHandler)
Patrick Devine's avatar
Patrick Devine committed
920
	r.POST("/api/copy", CopyModelHandler)
921
	r.DELETE("/api/delete", DeleteModelHandler)
Patrick Devine's avatar
Patrick Devine committed
922
	r.POST("/api/show", ShowModelHandler)
Michael Yang's avatar
Michael Yang committed
923
	r.POST("/api/blobs/:digest", CreateBlobHandler)
Michael Yang's avatar
Michael Yang committed
924
	r.HEAD("/api/blobs/:digest", HeadBlobHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
925

Michael Yang's avatar
Michael Yang committed
926
927
928
929
930
931
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

		r.Handle(method, "/api/tags", ListModelsHandler)
Michael Yang's avatar
Michael Yang committed
932
933
934
		r.Handle(method, "/api/version", func(c *gin.Context) {
			c.JSON(http.StatusOK, gin.H{"version": version.Version})
		})
Michael Yang's avatar
Michael Yang committed
935
936
	}

937
938
939
940
	return r
}

func Serve(ln net.Listener) error {
Michael Yang's avatar
Michael Yang committed
941
	level := slog.LevelInfo
942
	if debug := os.Getenv("OLLAMA_DEBUG"); debug != "" {
Michael Yang's avatar
Michael Yang committed
943
		level = slog.LevelDebug
944
	}
Michael Yang's avatar
Michael Yang committed
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960

	handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
		Level:     level,
		AddSource: true,
		ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr {
			if attr.Key == slog.SourceKey {
				source := attr.Value.Any().(*slog.Source)
				source.File = filepath.Base(source.File)
			}

			return attr
		},
	})

	slog.SetDefault(slog.New(handler))

961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
	if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		// clean up unused layers and manifests
		if err := PruneLayers(); err != nil {
			return err
		}

		manifestsPath, err := GetManifestPath()
		if err != nil {
			return err
		}

		if err := PruneDirectory(manifestsPath); err != nil {
			return err
		}
	}

	s, err := NewServer()
	if err != nil {
		return err
	}
	r := s.GenerateRoutes()

983
	slog.Info(fmt.Sprintf("Listening on %s (version %s)", ln.Addr(), version.Version))
984
	srvr := &http.Server{
Jeffrey Morgan's avatar
Jeffrey Morgan committed
985
986
987
		Handler: r,
	}

988
989
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
990
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
991
992
	go func() {
		<-signals
993
994
		if loaded.runner != nil {
			loaded.runner.Close()
995
		}
996
		os.RemoveAll(s.WorkDir)
997
998
999
		os.Exit(0)
	}()

1000
1001
1002
1003
	if err := llm.Init(s.WorkDir); err != nil {
		return fmt.Errorf("unable to initialize llm library %w", err)
	}
	if runtime.GOOS == "linux" { // TODO - windows too
1004
		// check compatibility to log warnings
1005
		if _, err := gpu.CheckVRAM(); err != nil {
1006
			slog.Info(err.Error())
1007
1008
1009
		}
	}

1010
	return srvr.Serve(ln)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1011
}
Michael Yang's avatar
Michael Yang committed
1012

1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
			if errorMsg, ok := r["error"].(string); ok {
				c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
				return
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in progress response"})
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
1038
func streamResponse(c *gin.Context, ch chan any) {
1039
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
1040
1041
1042
1043
1044
1045
1046
1047
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
1048
			slog.Info(fmt.Sprintf("streamResponse: json.Marshal failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1049
1050
1051
			return false
		}

1052
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
1053
1054
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
1055
			slog.Info(fmt.Sprintf("streamResponse: w.Write failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1056
1057
1058
1059
1060
1061
			return false
		}

		return true
	})
}
Bruce MacDonald's avatar
Bruce MacDonald committed
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089

func ChatHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()

	var req api.ChatRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	// validate the request
	switch {
	case req.Model == "":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	case len(req.Format) > 0 && req.Format != "json":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
		return
	}

1090
	model, err := GetModel(req.Model)
Bruce MacDonald's avatar
Bruce MacDonald committed
1091
1092
	if err != nil {
		var pErr *fs.PathError
1093
		if errors.As(err, &pErr) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1094
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
1095
1096
1097
1098
1099
1100
1101
1102
1103
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	opts, err := modelOptions(model, req.Options)
	if err != nil {
		if errors.Is(err, api.ErrInvalidOpts) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1104
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
1105
			return
Bruce MacDonald's avatar
Bruce MacDonald committed
1106
		}
1107
1108
1109
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
1110
1111
1112
1113
1114
1115
1116
1117

	var sessionDuration time.Duration
	if req.KeepAlive == nil {
		sessionDuration = defaultSessionDuration
	} else {
		sessionDuration = req.KeepAlive.Duration
	}

1118
1119
	if err := load(c, model, opts, sessionDuration); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
1120
1121
1122
1123
1124
		return
	}

	// an empty request loads the model
	if len(req.Messages) == 0 {
1125
1126
1127
1128
1129
1130
1131
		resp := api.ChatResponse{
			CreatedAt: time.Now().UTC(),
			Model:     req.Model,
			Done:      true,
			Message:   api.Message{Role: "assistant"},
		}
		c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
1132
1133
1134
1135
1136
		return
	}

	checkpointLoaded := time.Now()

1137
	chat, err := model.ChatPrompts(req.Messages)
Bruce MacDonald's avatar
Bruce MacDonald committed
1138
1139
1140
1141
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
1142
1143
1144
1145
1146
	prompt, err := trimmedPrompt(c.Request.Context(), chat, model)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
1147

1148
1149
	slog.Debug(fmt.Sprintf("prompt: %s", prompt))

Bruce MacDonald's avatar
Bruce MacDonald committed
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
	ch := make(chan any)

	go func() {
		defer close(ch)

		fn := func(r llm.PredictResult) {
			// Update model expiration
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)

			resp := api.ChatResponse{
1161
				Model:     req.Model,
1162
				CreatedAt: time.Now().UTC(),
1163
				Message:   api.Message{Role: "assistant", Content: r.Content},
Bruce MacDonald's avatar
Bruce MacDonald committed
1164
1165
1166
1167
1168
1169
1170
1171
1172
				Done:      r.Done,
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
			}

1173
1174
1175
			if r.Done {
				resp.TotalDuration = time.Since(checkpointStart)
				resp.LoadDuration = checkpointLoaded.Sub(checkpointStart)
Bruce MacDonald's avatar
Bruce MacDonald committed
1176
1177
1178
1179
1180
1181
1182
			}

			ch <- resp
		}

		// Start prediction
		predictReq := llm.PredictOpts{
1183
1184
			Prompt:  prompt,
			Format:  req.Format,
1185
			Images:  chat.CurrentImages,
1186
			Options: opts,
Bruce MacDonald's avatar
Bruce MacDonald committed
1187
1188
1189
1190
1191
1192
1193
		}
		if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
1194
1195
		// Accumulate responses into the final response
		var final api.ChatResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
1196
1197
		var sb strings.Builder
		for resp := range ch {
1198
1199
			switch r := resp.(type) {
			case api.ChatResponse:
1200
				sb.WriteString(r.Message.Content)
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
				final = r
			case gin.H:
				if errorMsg, ok := r["error"].(string); ok {
					c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
					return
				} else {
					c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
					return
				}
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
1213
1214
			}
		}
1215

1216
		final.Message = api.Message{Role: "assistant", Content: sb.String()}
1217
		c.JSON(http.StatusOK, final)
Bruce MacDonald's avatar
Bruce MacDonald committed
1218
1219
1220
1221
1222
		return
	}

	streamResponse(c, ch)
}
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320

// promptInfo stores the variables used to template a prompt, and the token length of the resulting template for some model
type promptInfo struct {
	vars     PromptVars
	tokenLen int
}

// trimmedPrompt builds a prompt to send to a running model. It ensures the prompt fits within the max context length,
// while preserving the most recent system message.
func trimmedPrompt(ctx context.Context, chat *ChatHistory, model *Model) (string, error) {
	if len(chat.Prompts) == 0 {
		return "", nil
	}

	var promptsToAdd []promptInfo
	var totalTokenLength int
	var systemPromptIncluded bool

	// reverse iterate through the prompts to build the prompt string in a way that fits the max context length
	for i := len(chat.Prompts) - 1; i >= 0; i-- {
		promptText, err := promptString(model, chat.Prompts[i], i == len(chat.Prompts)-1)
		if err != nil {
			return "", err
		}

		encodedTokens, err := loaded.runner.Encode(ctx, promptText)
		if err != nil {
			return "", err
		}

		if totalTokenLength+len(encodedTokens) > loaded.NumCtx && i != len(chat.Prompts)-1 {
			break // reached max context length, stop adding more prompts
		}

		totalTokenLength += len(encodedTokens)
		systemPromptIncluded = systemPromptIncluded || chat.Prompts[i].System != ""
		promptsToAdd = append(promptsToAdd, promptInfo{vars: chat.Prompts[i], tokenLen: len(encodedTokens)})
	}

	// ensure the system prompt is included, if not already
	if chat.LastSystem != "" && !systemPromptIncluded {
		var err error
		promptsToAdd, err = includeSystemPrompt(ctx, chat.LastSystem, totalTokenLength, promptsToAdd)
		if err != nil {
			return "", err
		}
	}

	promptsToAdd[len(promptsToAdd)-1].vars.First = true

	// construct the final prompt string from the prompts which fit within the context window
	var result string
	for i, prompt := range promptsToAdd {
		promptText, err := promptString(model, prompt.vars, i == 0)
		if err != nil {
			return "", err
		}
		result = promptText + result
	}
	return result, nil
}

// promptString applies the model template to the prompt
func promptString(model *Model, vars PromptVars, isMostRecent bool) (string, error) {
	if isMostRecent {
		p, err := model.PreResponsePrompt(vars)
		if err != nil {
			return "", fmt.Errorf("pre-response template: %w", err)
		}
		return p, nil
	}
	p, err := Prompt(model.Template, vars)
	if err != nil {
		return "", err
	}
	return p, nil
}

// includeSystemPrompt adjusts the prompts to include the system prompt.
func includeSystemPrompt(ctx context.Context, systemPrompt string, totalTokenLength int, promptsToAdd []promptInfo) ([]promptInfo, error) {
	systemTokens, err := loaded.runner.Encode(ctx, systemPrompt)
	if err != nil {
		return nil, err
	}

	for i := len(promptsToAdd) - 1; i >= 0; i-- {
		if totalTokenLength+len(systemTokens) <= loaded.NumCtx {
			promptsToAdd[i].vars.System = systemPrompt
			return promptsToAdd[:i+1], nil
		}
		totalTokenLength -= promptsToAdd[i].tokenLen
	}

	// if got here, system did not fit anywhere, so return the most recent prompt with the system message set
	recent := promptsToAdd[len(promptsToAdd)-1]
	recent.vars.System = systemPrompt
	return []promptInfo{recent}, nil
}