routes.go 39.8 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
Michael Yang's avatar
Michael Yang committed
4
	"bytes"
Michael Yang's avatar
Michael Yang committed
5
	"cmp"
6
	"context"
7
	"encoding/binary"
Michael Yang's avatar
Michael Yang committed
8
	"encoding/json"
9
	"errors"
10
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
11
	"io"
12
	"log/slog"
13
	"math"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
14
15
	"net"
	"net/http"
16
	"net/netip"
17
	"os"
18
	"os/signal"
Michael Yang's avatar
Michael Yang committed
19
	"path/filepath"
20
	"slices"
Michael Yang's avatar
Michael Yang committed
21
	"strings"
22
	"syscall"
23
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
24

Michael Yang's avatar
Michael Yang committed
25
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
26
	"github.com/gin-gonic/gin"
27
	"golang.org/x/sync/errgroup"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
28

29
	"github.com/ollama/ollama/api"
30
	"github.com/ollama/ollama/discover"
31
	"github.com/ollama/ollama/envconfig"
32
33
	"github.com/ollama/ollama/llm"
	"github.com/ollama/ollama/openai"
34
	"github.com/ollama/ollama/parser"
35
	"github.com/ollama/ollama/runners"
36
	"github.com/ollama/ollama/server/imageproc"
Michael Yang's avatar
Michael Yang committed
37
	"github.com/ollama/ollama/template"
38
	"github.com/ollama/ollama/types/errtypes"
Michael Yang's avatar
Michael Yang committed
39
	"github.com/ollama/ollama/types/model"
40
	"github.com/ollama/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
41
42
)

Michael Yang's avatar
Michael Yang committed
43
44
var mode string = gin.DebugMode

45
type Server struct {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
46
47
	addr  net.Addr
	sched *Scheduler
48
49
}

Michael Yang's avatar
Michael Yang committed
50
51
52
53
54
55
56
57
58
59
60
61
func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Michael Yang's avatar
lint  
Michael Yang committed
62
63
64
65
var (
	errRequired    = errors.New("is required")
	errBadTemplate = errors.New("template error")
)
Michael Yang's avatar
Michael Yang committed
66

67
68
69
70
71
72
73
74
75
76
77
func modelOptions(model *Model, requestOpts map[string]interface{}) (api.Options, error) {
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		return api.Options{}, err
	}

	if err := opts.FromMap(requestOpts); err != nil {
		return api.Options{}, err
	}

	return opts, nil
Bruce MacDonald's avatar
Bruce MacDonald committed
78
79
}

Michael Yang's avatar
Michael Yang committed
80
81
82
// scheduleRunner schedules a runner after validating inputs such as capabilities and model options.
// It returns the allocated runner, model instance, and consolidated options if successful and error otherwise.
func (s *Server) scheduleRunner(ctx context.Context, name string, caps []Capability, requestOpts map[string]any, keepAlive *api.Duration) (llm.LlamaServer, *Model, *api.Options, error) {
Michael Yang's avatar
Michael Yang committed
83
	if name == "" {
Michael Yang's avatar
Michael Yang committed
84
		return nil, nil, nil, fmt.Errorf("model %w", errRequired)
Bruce MacDonald's avatar
Bruce MacDonald committed
85
86
	}

Michael Yang's avatar
Michael Yang committed
87
	model, err := GetModel(name)
Bruce MacDonald's avatar
Bruce MacDonald committed
88
	if err != nil {
Michael Yang's avatar
Michael Yang committed
89
		return nil, nil, nil, err
90
91
	}

Michael Yang's avatar
Michael Yang committed
92
	if err := model.CheckCapabilities(caps...); err != nil {
Michael Yang's avatar
Michael Yang committed
93
		return nil, nil, nil, fmt.Errorf("%s %w", name, err)
94
95
	}

Michael Yang's avatar
Michael Yang committed
96
	opts, err := modelOptions(model, requestOpts)
97
	if err != nil {
Michael Yang's avatar
Michael Yang committed
98
		return nil, nil, nil, err
99
100
	}

Michael Yang's avatar
Michael Yang committed
101
	runnerCh, errCh := s.sched.GetRunner(ctx, model, opts, keepAlive)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
102
103
	var runner *runnerRef
	select {
Michael Yang's avatar
Michael Yang committed
104
105
	case runner = <-runnerCh:
	case err = <-errCh:
Michael Yang's avatar
Michael Yang committed
106
		return nil, nil, nil, err
Bruce MacDonald's avatar
Bruce MacDonald committed
107
108
	}

Michael Yang's avatar
Michael Yang committed
109
	return runner.llama, model, &opts, nil
Michael Yang's avatar
Michael Yang committed
110
111
112
}

func (s *Server) GenerateHandler(c *gin.Context) {
113
	checkpointStart := time.Now()
Michael Yang's avatar
Michael Yang committed
114
115
116
117
118
119
	var req api.GenerateRequest
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
120
121
122
		return
	}

123
124
125
126
127
128
129
130
131
132
133
134
135
	model, err := GetModel(req.Model)
	if err != nil {
		switch {
		case os.IsNotExist(err):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		case err.Error() == "invalid model name":
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

Patrick Devine's avatar
Patrick Devine committed
136
137
138
139
140
141
142
143
144
145
146
147
148
149
	// expire the runner
	if req.Prompt == "" && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
		s.sched.expireRunner(model)

		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Response:   "",
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

150
	if req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0) {
Michael Yang's avatar
Michael Yang committed
151
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
Michael Yang's avatar
Michael Yang committed
152
153
154
		return
	}

Michael Yang's avatar
Michael Yang committed
155
	caps := []Capability{CapabilityCompletion}
156
157
158
159
	if req.Suffix != "" {
		caps = append(caps, CapabilityInsert)
	}

Michael Yang's avatar
Michael Yang committed
160
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
161
162
163
164
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support generate", req.Model)})
		return
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
165
166
167
168
		handleScheduleError(c, req.Model, err)
		return
	}

169
170
	checkpointLoaded := time.Now()

171
	// load the model
Michael Yang's avatar
Michael Yang committed
172
173
174
175
176
177
178
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Done:       true,
			DoneReason: "load",
		})
Michael Yang's avatar
Michael Yang committed
179
180
		return
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
181

182
183
184
185
186
187
	isMllama := checkMllamaModelFamily(model)
	if isMllama && len(req.Images) > 1 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "this model only supports one image: more than one image sent"})
		return
	}

Michael Yang's avatar
Michael Yang committed
188
189
	images := make([]llm.ImageData, len(req.Images))
	for i := range req.Images {
190
191
192
193
194
195
196
197
198
199
200
201
202
203
		if isMllama {
			data, aspectRatioID, err := imageproc.Preprocess(req.Images[i])
			if err != nil {
				c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "error processing image"})
				return
			}

			buf := new(bytes.Buffer)
			err = binary.Write(buf, binary.LittleEndian, data)
			if err != nil {
				c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "error processing image"})
				return
			}

204
			images[i] = llm.ImageData{ID: i, Data: buf.Bytes(), AspectRatioID: aspectRatioID}
205
206
207
		} else {
			images[i] = llm.ImageData{ID: i, Data: req.Images[i]}
		}
Michael Yang's avatar
Michael Yang committed
208
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
209

Michael Yang's avatar
Michael Yang committed
210
211
	prompt := req.Prompt
	if !req.Raw {
Michael Yang's avatar
Michael Yang committed
212
		tmpl := m.Template
Michael Yang's avatar
Michael Yang committed
213
214
215
216
217
218
219
220
		if req.Template != "" {
			tmpl, err = template.Parse(req.Template)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

221
222
223
224
225
226
227
228
229
230
231
232
		var values template.Values
		if req.Suffix != "" {
			values.Prompt = prompt
			values.Suffix = req.Suffix
		} else {
			var msgs []api.Message
			if req.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: req.System})
			} else if m.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: m.System})
			}

Michael Yang's avatar
Michael Yang committed
233
234
235
236
			if req.Context == nil {
				msgs = append(msgs, m.Messages...)
			}

237
			for _, i := range images {
238
				imgPrompt := ""
239
				if isMllama {
240
					imgPrompt = "<|image|>"
241
				}
242
				msgs = append(msgs, api.Message{Role: "user", Content: fmt.Sprintf("[img-%d]"+imgPrompt, i.ID)})
243
244
245
246
247
			}

			values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt})
		}

Michael Yang's avatar
Michael Yang committed
248
249
		var b bytes.Buffer
		if req.Context != nil {
250
			slog.Warn("the context field is deprecated and will be removed in a future version of Ollama")
251
			s, err := r.Detokenize(c.Request.Context(), req.Context)
Michael Yang's avatar
Michael Yang committed
252
253
254
255
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
256
			b.WriteString(s)
Michael Yang's avatar
Michael Yang committed
257
		}
258
259
260
261
262
263
264

		if err := tmpl.Execute(&b, values); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		prompt = b.String()
Bruce MacDonald's avatar
Bruce MacDonald committed
265
266
	}

267
	slog.Debug("generate request", "images", len(images), "prompt", prompt)
268

Bruce MacDonald's avatar
Bruce MacDonald committed
269
270
	ch := make(chan any)
	go func() {
271
272
		// TODO (jmorganca): avoid building the response twice both here and below
		var sb strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
273
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
274
		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
Michael Yang's avatar
Michael Yang committed
275
276
			Prompt:  prompt,
			Images:  images,
277
			Format:  req.Format,
Michael Yang's avatar
Michael Yang committed
278
			Options: opts,
279
280
		}, func(cr llm.CompletionResponse) {
			res := api.GenerateResponse{
281
282
				Model:      req.Model,
				CreatedAt:  time.Now().UTC(),
283
284
285
				Response:   cr.Content,
				Done:       cr.Done,
				DoneReason: cr.DoneReason,
Bruce MacDonald's avatar
Bruce MacDonald committed
286
				Metrics: api.Metrics{
287
288
289
290
					PromptEvalCount:    cr.PromptEvalCount,
					PromptEvalDuration: cr.PromptEvalDuration,
					EvalCount:          cr.EvalCount,
					EvalDuration:       cr.EvalDuration,
Bruce MacDonald's avatar
Bruce MacDonald committed
291
				},
Bruce MacDonald's avatar
Bruce MacDonald committed
292
			}
293
294
295
296
297
298
299
300
301
302

			if _, err := sb.WriteString(cr.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
			}

			if cr.Done {
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)

				if !req.Raw {
303
					tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String())
304
305
306
307
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
308
					res.Context = tokens
309
310
311
312
				}
			}

			ch <- res
Michael Yang's avatar
Michael Yang committed
313
		}); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
314
315
316
317
318
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
Michael Yang committed
319
		var r api.GenerateResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
320
		var sb strings.Builder
Michael Yang's avatar
Michael Yang committed
321
322
		for rr := range ch {
			switch t := rr.(type) {
323
			case api.GenerateResponse:
Michael Yang's avatar
Michael Yang committed
324
325
				sb.WriteString(t.Response)
				r = t
326
			case gin.H:
Michael Yang's avatar
Michael Yang committed
327
328
329
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
330
				}
Michael Yang's avatar
Michael Yang committed
331
332
333

				c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
				return
334
			default:
Michael Yang's avatar
Michael Yang committed
335
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
Bruce MacDonald's avatar
Bruce MacDonald committed
336
337
338
				return
			}
		}
339

Michael Yang's avatar
Michael Yang committed
340
341
		r.Response = sb.String()
		c.JSON(http.StatusOK, r)
Bruce MacDonald's avatar
Bruce MacDonald committed
342
343
344
345
346
347
		return
	}

	streamResponse(c, ch)
}

348
func (s *Server) EmbedHandler(c *gin.Context) {
349
	checkpointStart := time.Now()
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
	var req api.EmbedRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	truncate := true

	if req.Truncate != nil && !*req.Truncate {
		truncate = false
	}

	var input []string

	switch i := req.Input.(type) {
	case string:
		if len(i) > 0 {
			input = append(input, i)
		}
	case []any:
		for _, v := range i {
			if _, ok := v.(string); !ok {
				c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
				return
			}
			input = append(input, v.(string))
		}
	default:
383
384
385
386
		if req.Input != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
			return
		}
387
388
389
390
391
392
393
394
	}

	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}

395
396
	checkpointLoaded := time.Now()

397
398
399
400
401
	if len(input) == 0 {
		c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
		return
	}

402
403
404
405
406
407
	kvData, err := getKVData(m.ModelPath, false)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

408
	var count int
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
	for i, s := range input {
		tokens, err := r.Tokenize(c.Request.Context(), s)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		ctxLen := min(opts.NumCtx, int(kvData.ContextLength()))
		if len(tokens) > ctxLen {
			if !truncate {
				c.JSON(http.StatusBadRequest, gin.H{"error": "input length exceeds maximum context length"})
				return
			}

			tokens = tokens[:ctxLen]
			s, err = r.Detokenize(c.Request.Context(), tokens)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

431
432
		count += len(tokens)

433
434
		input[i] = s
	}
435
436
437
438
439
440
441
442
443
444
445
446

	var g errgroup.Group
	embeddings := make([][]float32, len(input))
	for i, text := range input {
		g.Go(func() error {
			embedding, err := r.Embedding(c.Request.Context(), text)
			if err != nil {
				return err
			}
			embeddings[i] = normalize(embedding)
			return nil
		})
447
448
	}

449
450
451
452
	if err := g.Wait(); err != nil {
		slog.Error("embedding generation failed", "error", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embeddings: %v", err)})
		return
453
454
455
	}

	resp := api.EmbedResponse{
456
		Model:           req.Model,
457
		Embeddings:      embeddings,
458
459
		TotalDuration:   time.Since(checkpointStart),
		LoadDuration:    checkpointLoaded.Sub(checkpointStart),
460
		PromptEvalCount: count,
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
	}
	c.JSON(http.StatusOK, resp)
}

func normalize(vec []float32) []float32 {
	var sum float32
	for _, v := range vec {
		sum += v * v
	}

	norm := float32(0.0)
	if sum > 0 {
		norm = float32(1.0 / math.Sqrt(float64(sum)))
	}

	for i := range vec {
		vec[i] *= norm
	}
	return vec
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
482
func (s *Server) EmbeddingsHandler(c *gin.Context) {
Bruce MacDonald's avatar
Bruce MacDonald committed
483
	var req api.EmbeddingRequest
Michael Yang's avatar
Michael Yang committed
484
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
485
486
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
487
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
488
489
490
491
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
492
	r, _, _, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
Bruce MacDonald's avatar
Bruce MacDonald committed
493
	if err != nil {
Michael Yang's avatar
Michael Yang committed
494
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
495
496
497
		return
	}

498
499
500
	// an empty request loads the model
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float64{}})
Bruce MacDonald's avatar
Bruce MacDonald committed
501
502
503
		return
	}

504
	embedding, err := r.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
505
	if err != nil {
506
		slog.Info(fmt.Sprintf("embedding generation failed: %v", err))
507
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embedding: %v", err)})
Bruce MacDonald's avatar
Bruce MacDonald committed
508
509
510
		return
	}

511
512
513
	var e []float64
	for _, v := range embedding {
		e = append(e, float64(v))
514
515
516
	}

	resp := api.EmbeddingResponse{
517
		Embedding: e,
518
519
	}
	c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
520
521
}

522
func (s *Server) PullHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
523
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
524
525
526
527
528
529
530
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
531
532
533
		return
	}

534
535
536
537
538
539
	name := model.ParseName(cmp.Or(req.Model, req.Name))
	if !name.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid model name"})
		return
	}

540
541
	name, err = getExistingName(name)
	if err != nil {
542
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
543
544
545
		return
	}

546
547
548
	ch := make(chan any)
	go func() {
		defer close(ch)
549
550
		fn := func(r api.ProgressResponse) {
			ch <- r
551
		}
552

Michael Yang's avatar
Michael Yang committed
553
		regOpts := &registryOptions{
554
555
556
			Insecure: req.Insecure,
		}

557
558
559
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

560
		if err := PullModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
561
			ch <- gin.H{"error": err.Error()}
562
563
564
		}
	}()

565
566
567
568
569
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

570
571
572
	streamResponse(c, ch)
}

573
func (s *Server) PushHandler(c *gin.Context) {
574
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
575
576
577
578
579
580
581
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
582
583
		return
	}
Michael Yang's avatar
Michael Yang committed
584

Michael Yang's avatar
Michael Yang committed
585
586
587
588
589
590
591
	var model string
	if req.Model != "" {
		model = req.Model
	} else if req.Name != "" {
		model = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
592
593
594
		return
	}

595
596
597
	ch := make(chan any)
	go func() {
		defer close(ch)
598
599
		fn := func(r api.ProgressResponse) {
			ch <- r
600
		}
601

Michael Yang's avatar
Michael Yang committed
602
		regOpts := &registryOptions{
603
604
605
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
606
607
608
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

Michael Yang's avatar
Michael Yang committed
609
		if err := PushModel(ctx, model, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
610
			ch <- gin.H{"error": err.Error()}
611
612
613
		}
	}()

614
615
616
617
618
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

619
620
621
	streamResponse(c, ch)
}

622
623
624
625
626
// getExistingName returns the original, on disk name if the input name is a
// case-insensitive match, otherwise it returns the input name.
func getExistingName(n model.Name) (model.Name, error) {
	var zero model.Name
	existing, err := Manifests(true)
627
	if err != nil {
628
		return zero, err
629
	}
630
631
632
	for e := range existing {
		if n.EqualFold(e) {
			return e, nil
633
634
		}
	}
635
	return n, nil
636
637
}

638
func (s *Server) CreateHandler(c *gin.Context) {
639
640
	var r api.CreateRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
641
642
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
643
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
644
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
645
		return
646
647
	}

648
	name := model.ParseName(cmp.Or(r.Model, r.Name))
Michael Yang's avatar
Michael Yang committed
649
	if !name.IsValid() {
650
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": errtypes.InvalidModelNameErrMsg})
651
652
653
		return
	}

654
655
	name, err := getExistingName(name)
	if err != nil {
656
657
658
659
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

660
	if r.Path == "" && r.Modelfile == "" {
Michael Yang's avatar
Michael Yang committed
661
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"})
Michael Yang's avatar
Michael Yang committed
662
663
		return
	}
Michael Yang's avatar
Michael Yang committed
664

665
666
667
	var sr io.Reader = strings.NewReader(r.Modelfile)
	if r.Path != "" && r.Modelfile == "" {
		f, err := os.Open(r.Path)
Michael Yang's avatar
Michael Yang committed
668
669
670
671
		if err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)})
			return
		}
Michael Yang's avatar
Michael Yang committed
672
		defer f.Close()
Michael Yang's avatar
Michael Yang committed
673

674
		sr = f
Michael Yang's avatar
Michael Yang committed
675
	}
Michael Yang's avatar
Michael Yang committed
676

677
	f, err := parser.ParseFile(sr)
Michael Yang's avatar
Michael Yang committed
678
679
680
681
682
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
683
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
684
685
	go func() {
		defer close(ch)
686
687
		fn := func(resp api.ProgressResponse) {
			ch <- resp
688
689
		}

690
691
692
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

693
		quantization := cmp.Or(r.Quantize, r.Quantization)
Josh's avatar
Josh committed
694
695
696
		if err := CreateModel(ctx, name, filepath.Dir(r.Path), strings.ToUpper(quantization), f, fn); errors.Is(err, errBadTemplate) {
			ch <- gin.H{"error": err.Error(), "status": http.StatusBadRequest}
		} else if err != nil {
Michael Yang's avatar
Michael Yang committed
697
			ch <- gin.H{"error": err.Error()}
698
		}
Michael Yang's avatar
Michael Yang committed
699
	}()
Michael Yang's avatar
Michael Yang committed
700

701
	if r.Stream != nil && !*r.Stream {
702
703
704
705
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
706
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
707
708
}

709
func (s *Server) DeleteHandler(c *gin.Context) {
710
711
	var r api.DeleteRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
712
713
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
714
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
715
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
716
717
718
		return
	}

719
720
721
	n := model.ParseName(cmp.Or(r.Model, r.Name))
	if !n.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("name %q is invalid", cmp.Or(r.Model, r.Name))})
722
723
		return
	}
Michael Yang's avatar
Michael Yang committed
724

725
	m, err := ParseNamedManifest(n)
Michael Yang's avatar
Michael Yang committed
726
	if err != nil {
727
728
729
730
731
732
		switch {
		case os.IsNotExist(err):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", cmp.Or(r.Model, r.Name))})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
Michael Yang's avatar
Michael Yang committed
733
734
735
		return
	}

736
	if err := m.Remove(); err != nil {
Michael Yang's avatar
Michael Yang committed
737
738
739
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
740
741
742
743
744

	if err := m.RemoveLayers(); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
745
746
}

747
func (s *Server) ShowHandler(c *gin.Context) {
Patrick Devine's avatar
Patrick Devine committed
748
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
749
750
751
752
753
754
755
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
756
757
758
		return
	}

Michael Yang's avatar
Michael Yang committed
759
	if req.Model != "" {
Michael Yang's avatar
Michael Yang committed
760
		// noop
Michael Yang's avatar
Michael Yang committed
761
	} else if req.Name != "" {
Michael Yang's avatar
Michael Yang committed
762
		req.Model = req.Name
Michael Yang's avatar
Michael Yang committed
763
	} else {
764
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
765
766
767
		return
	}

768
	resp, err := GetModelInfo(req)
Patrick Devine's avatar
Patrick Devine committed
769
	if err != nil {
770
771
		switch {
		case os.IsNotExist(err):
Michael Yang's avatar
Michael Yang committed
772
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
773
774
775
		case err.Error() == "invalid model name":
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
Patrick Devine's avatar
Patrick Devine committed
776
777
778
779
780
781
782
783
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

784
func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
785
	m, err := GetModel(req.Model)
Patrick Devine's avatar
Patrick Devine committed
786
787
788
789
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
790
	modelDetails := api.ModelDetails{
791
792
793
794
795
796
		ParentModel:       m.ParentModel,
		Format:            m.Config.ModelFormat,
		Family:            m.Config.ModelFamily,
		Families:          m.Config.ModelFamilies,
		ParameterSize:     m.Config.ModelType,
		QuantizationLevel: m.Config.FileType,
Patrick Devine's avatar
Patrick Devine committed
797
798
	}

799
	if req.System != "" {
800
		m.System = req.System
801
802
	}

Michael Yang's avatar
Michael Yang committed
803
804
805
	msgs := make([]api.Message, len(m.Messages))
	for i, msg := range m.Messages {
		msgs[i] = api.Message{Role: msg.Role, Content: msg.Content}
806
807
	}

808
809
	n := model.ParseName(req.Model)
	if !n.IsValid() {
Michael Yang's avatar
lint  
Michael Yang committed
810
		return nil, errors.New("invalid model name")
811
812
813
814
815
816
817
	}

	manifest, err := ParseNamedManifest(n)
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
818
	resp := &api.ShowResponse{
819
820
		License:    strings.Join(m.License, "\n"),
		System:     m.System,
Michael Yang's avatar
Michael Yang committed
821
		Template:   m.Template.String(),
822
823
824
		Details:    modelDetails,
		Messages:   msgs,
		ModifiedAt: manifest.fi.ModTime(),
Patrick Devine's avatar
Patrick Devine committed
825
826
827
828
	}

	var params []string
	cs := 30
829
	for k, v := range m.Options {
Patrick Devine's avatar
Patrick Devine committed
830
831
832
		switch val := v.(type) {
		case []interface{}:
			for _, nv := range val {
Patrick Devine's avatar
Patrick Devine committed
833
				params = append(params, fmt.Sprintf("%-*s %#v", cs, k, nv))
Patrick Devine's avatar
Patrick Devine committed
834
			}
Patrick Devine's avatar
Patrick Devine committed
835
836
		default:
			params = append(params, fmt.Sprintf("%-*s %#v", cs, k, v))
Patrick Devine's avatar
Patrick Devine committed
837
838
839
840
		}
	}
	resp.Parameters = strings.Join(params, "\n")

841
842
	for k, v := range req.Options {
		if _, ok := req.Options[k]; ok {
843
			m.Options[k] = v
844
845
846
		}
	}

847
	var sb strings.Builder
848
	fmt.Fprintln(&sb, "# Modelfile generated by \"ollama show\"")
849
	fmt.Fprintln(&sb, "# To build a new Modelfile based on this, replace FROM with:")
850
851
	fmt.Fprintf(&sb, "# FROM %s\n\n", m.ShortName)
	fmt.Fprint(&sb, m.String())
852
	resp.Modelfile = sb.String()
853

854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
	kvData, err := getKVData(m.ModelPath, req.Verbose)
	if err != nil {
		return nil, err
	}
	delete(kvData, "general.name")
	delete(kvData, "tokenizer.chat_template")
	resp.ModelInfo = kvData

	if len(m.ProjectorPaths) > 0 {
		projectorData, err := getKVData(m.ProjectorPaths[0], req.Verbose)
		if err != nil {
			return nil, err
		}
		resp.ProjectorInfo = projectorData
	}

Patrick Devine's avatar
Patrick Devine committed
870
871
872
	return resp, nil
}

873
func getKVData(digest string, verbose bool) (llm.KV, error) {
874
875
876
877
878
	maxArraySize := 0
	if verbose {
		maxArraySize = -1
	}
	kvData, err := llm.LoadModel(digest, maxArraySize)
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
	if err != nil {
		return nil, err
	}

	kv := kvData.KV()

	if !verbose {
		for k := range kv {
			if t, ok := kv[k].([]any); len(t) > 5 && ok {
				kv[k] = []any{}
			}
		}
	}

	return kv, nil
}

896
func (s *Server) ListHandler(c *gin.Context) {
897
	ms, err := Manifests(true)
Patrick Devine's avatar
Patrick Devine committed
898
899
900
901
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
902

903
	models := []api.ListModelResponse{}
904
905
	for n, m := range ms {
		var cf ConfigV2
906
907
908
909
910
911
912
913
914
915
916
917
918

		if m.Config.Digest != "" {
			f, err := m.Config.Open()
			if err != nil {
				slog.Warn("bad manifest filepath", "name", n, "error", err)
				continue
			}
			defer f.Close()

			if err := json.NewDecoder(f).Decode(&cf); err != nil {
				slog.Warn("bad manifest config", "name", n, "error", err)
				continue
			}
Patrick Devine's avatar
Patrick Devine committed
919
		}
Michael Yang's avatar
Michael Yang committed
920

921
		// tag should never be masked
922
		models = append(models, api.ListModelResponse{
923
924
925
926
927
928
929
930
931
932
933
934
935
			Model:      n.DisplayShortest(),
			Name:       n.DisplayShortest(),
			Size:       m.Size(),
			Digest:     m.digest,
			ModifiedAt: m.fi.ModTime(),
			Details: api.ModelDetails{
				Format:            cf.ModelFormat,
				Family:            cf.ModelFamily,
				Families:          cf.ModelFamilies,
				ParameterSize:     cf.ModelType,
				QuantizationLevel: cf.FileType,
			},
		})
Patrick Devine's avatar
Patrick Devine committed
936
937
	}

938
	slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
939
940
941
942
		// most recently modified first
		return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
	})

Michael Yang's avatar
Michael Yang committed
943
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
944
945
}

946
func (s *Server) CopyHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
947
948
	var r api.CopyRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
949
950
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
951
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
952
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
953
954
955
		return
	}

Michael Yang's avatar
Michael Yang committed
956
957
	src := model.ParseName(r.Source)
	if !src.IsValid() {
Michael Yang's avatar
Michael Yang committed
958
959
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("source %q is invalid", r.Source)})
		return
960
	}
961
962
963
964
965
	src, err := getExistingName(src)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
966

Michael Yang's avatar
Michael Yang committed
967
968
	dst := model.ParseName(r.Destination)
	if !dst.IsValid() {
969
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("destination %q is invalid", r.Destination)})
Patrick Devine's avatar
Patrick Devine committed
970
971
		return
	}
972
973
	dst, err = getExistingName(dst)
	if err != nil {
974
975
976
977
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
978
979
980
981
982
	if err := CopyModel(src, dst); errors.Is(err, os.ErrNotExist) {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
	} else if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
	}
Patrick Devine's avatar
Patrick Devine committed
983
984
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
985
func (s *Server) HeadBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
986
987
988
989
990
991
992
993
994
995
996
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); err != nil {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	}

Michael Yang's avatar
Michael Yang committed
997
	c.Status(http.StatusOK)
Michael Yang's avatar
Michael Yang committed
998
999
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1000
func (s *Server) CreateBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
1001
1002
	if ib, ok := intermediateBlobs[c.Param("digest")]; ok {
		p, err := GetBlobsPath(ib)
1003
1004
1005
1006
1007
1008
		if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
Michael Yang's avatar
Michael Yang committed
1009
1010
			slog.Info("evicting intermediate blob which no longer exists", "digest", ib)
			delete(intermediateBlobs, c.Param("digest"))
1011
1012
1013
1014
1015
1016
1017
1018
1019
		} else if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		} else {
			c.Status(http.StatusOK)
			return
		}
	}

1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	_, err = os.Stat(path)
	switch {
	case errors.Is(err, os.ErrNotExist):
		// noop
	case err != nil:
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	default:
		c.Status(http.StatusOK)
		return
	}

1038
	layer, err := NewLayer(c.Request.Body, "")
Michael Yang's avatar
Michael Yang committed
1039
1040
1041
1042
1043
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

1044
1045
	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
Michael Yang's avatar
Michael Yang committed
1046
1047
1048
		return
	}

Michael Yang's avatar
Michael Yang committed
1049
	c.Status(http.StatusCreated)
Michael Yang's avatar
Michael Yang committed
1050
1051
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
func isLocalIP(ip netip.Addr) bool {
	if interfaces, err := net.Interfaces(); err == nil {
		for _, iface := range interfaces {
			addrs, err := iface.Addrs()
			if err != nil {
				continue
			}

			for _, a := range addrs {
				if parsed, _, err := net.ParseCIDR(a.String()); err == nil {
					if parsed.String() == ip.String() {
						return true
					}
				}
			}
		}
	}

	return false
}

1073
func allowedHost(host string) bool {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1074
	if host == "" || host == "localhost" {
1075
1076
1077
1078
1079
1080
1081
		return true
	}

	if hostname, err := os.Hostname(); err == nil && host == hostname {
		return true
	}

Michael Yang's avatar
lint  
Michael Yang committed
1082
	tlds := []string{
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1083
1084
1085
		"localhost",
		"local",
		"internal",
1086
	}
1087

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1088
	// check if the host is a local TLD
1089
1090
1091
1092
1093
1094
	for _, tld := range tlds {
		if strings.HasSuffix(host, "."+tld) {
			return true
		}
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1095
	return false
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1096
}
1097

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1098
1099
1100
func allowedHostsMiddleware(addr net.Addr) gin.HandlerFunc {
	return func(c *gin.Context) {
		if addr == nil {
1101
1102
1103
1104
			c.Next()
			return
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1105
		if addr, err := netip.ParseAddrPort(addr.String()); err == nil && !addr.Addr().IsLoopback() {
1106
1107
1108
1109
1110
1111
1112
1113
1114
			c.Next()
			return
		}

		host, _, err := net.SplitHostPort(c.Request.Host)
		if err != nil {
			host = c.Request.Host
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1115
		if addr, err := netip.ParseAddr(host); err == nil {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1116
			if addr.IsLoopback() || addr.IsPrivate() || addr.IsUnspecified() || isLocalIP(addr) {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1117
1118
1119
1120
1121
				c.Next()
				return
			}
		}

1122
		if allowedHost(host) {
Michael Yang's avatar
lint  
Michael Yang committed
1123
			if c.Request.Method == http.MethodOptions {
1124
1125
1126
1127
				c.AbortWithStatus(http.StatusNoContent)
				return
			}

1128
1129
1130
1131
1132
1133
			c.Next()
			return
		}

		c.AbortWithStatus(http.StatusForbidden)
	}
1134
}
1135

1136
func (s *Server) GenerateRoutes() http.Handler {
Michael Yang's avatar
Michael Yang committed
1137
1138
	config := cors.DefaultConfig()
	config.AllowWildcard = true
1139
	config.AllowBrowserExtensions = true
1140
	config.AllowHeaders = []string{"Authorization", "Content-Type", "User-Agent", "Accept", "X-Requested-With"}
1141
	openAIProperties := []string{"lang", "package-version", "os", "arch", "retry-count", "runtime", "runtime-version", "async"}
royjhan's avatar
royjhan committed
1142
1143
1144
	for _, prop := range openAIProperties {
		config.AllowHeaders = append(config.AllowHeaders, "x-stainless-"+prop)
	}
Michael Yang's avatar
origins  
Michael Yang committed
1145
	config.AllowOrigins = envconfig.Origins()
Michael Yang's avatar
Michael Yang committed
1146

Bruce MacDonald's avatar
Bruce MacDonald committed
1147
	r := gin.Default()
1148
1149
	r.Use(
		cors.New(config),
1150
		allowedHostsMiddleware(s.addr),
1151
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
1152

1153
	r.POST("/api/pull", s.PullHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1154
1155
	r.POST("/api/generate", s.GenerateHandler)
	r.POST("/api/chat", s.ChatHandler)
1156
	r.POST("/api/embed", s.EmbedHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1157
	r.POST("/api/embeddings", s.EmbeddingsHandler)
1158
1159
1160
1161
1162
	r.POST("/api/create", s.CreateHandler)
	r.POST("/api/push", s.PushHandler)
	r.POST("/api/copy", s.CopyHandler)
	r.DELETE("/api/delete", s.DeleteHandler)
	r.POST("/api/show", s.ShowHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1163
1164
	r.POST("/api/blobs/:digest", s.CreateBlobHandler)
	r.HEAD("/api/blobs/:digest", s.HeadBlobHandler)
1165
	r.GET("/api/ps", s.PsHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1166

1167
	// Compatibility endpoints
1168
	r.POST("/v1/chat/completions", openai.ChatMiddleware(), s.ChatHandler)
1169
	r.POST("/v1/completions", openai.CompletionsMiddleware(), s.GenerateHandler)
1170
	r.POST("/v1/embeddings", openai.EmbeddingsMiddleware(), s.EmbedHandler)
1171
1172
	r.GET("/v1/models", openai.ListMiddleware(), s.ListHandler)
	r.GET("/v1/models/:model", openai.RetrieveMiddleware(), s.ShowHandler)
1173

Michael Yang's avatar
Michael Yang committed
1174
1175
1176
1177
1178
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

1179
		r.Handle(method, "/api/tags", s.ListHandler)
Michael Yang's avatar
Michael Yang committed
1180
1181
1182
		r.Handle(method, "/api/version", func(c *gin.Context) {
			c.JSON(http.StatusOK, gin.H{"version": version.Version})
		})
Michael Yang's avatar
Michael Yang committed
1183
1184
	}

1185
1186
1187
1188
	return r
}

func Serve(ln net.Listener) error {
Michael Yang's avatar
Michael Yang committed
1189
	level := slog.LevelInfo
Michael Yang's avatar
Michael Yang committed
1190
	if envconfig.Debug() {
Michael Yang's avatar
Michael Yang committed
1191
		level = slog.LevelDebug
1192
	}
Michael Yang's avatar
Michael Yang committed
1193

1194
	slog.Info("server config", "env", envconfig.Values())
Michael Yang's avatar
Michael Yang committed
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
	handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
		Level:     level,
		AddSource: true,
		ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr {
			if attr.Key == slog.SourceKey {
				source := attr.Value.Any().(*slog.Source)
				source.File = filepath.Base(source.File)
			}

			return attr
		},
	})

	slog.SetDefault(slog.New(handler))

1210
1211
1212
1213
1214
1215
1216
1217
	blobsDir, err := GetBlobsPath("")
	if err != nil {
		return err
	}
	if err := fixBlobs(blobsDir); err != nil {
		return err
	}

Michael Yang's avatar
bool  
Michael Yang committed
1218
	if !envconfig.NoPrune() {
1219
1220
1221
1222
1223
1224
1225
		if _, err := Manifests(false); err != nil {
			slog.Warn("corrupt manifests detected, skipping prune operation.  Re-pull or delete to clear", "error", err)
		} else {
			// clean up unused layers and manifests
			if err := PruneLayers(); err != nil {
				return err
			}
1226

1227
1228
1229
1230
			manifestsPath, err := GetManifestPath()
			if err != nil {
				return err
			}
1231

1232
1233
1234
			if err := PruneDirectory(manifestsPath); err != nil {
				return err
			}
1235
1236
1237
		}
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1238
	ctx, done := context.WithCancel(context.Background())
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1239
1240
	schedCtx, schedDone := context.WithCancel(ctx)
	sched := InitScheduler(schedCtx)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1241
	s := &Server{addr: ln.Addr(), sched: sched}
1242
1243

	http.Handle("/", s.GenerateRoutes())
1244

1245
	slog.Info(fmt.Sprintf("Listening on %s (version %s)", ln.Addr(), version.Version))
1246
	srvr := &http.Server{
1247
1248
1249
1250
1251
1252
1253
1254
1255
		// Use http.DefaultServeMux so we get net/http/pprof for
		// free.
		//
		// TODO(bmizerany): Decide if we want to make this
		// configurable so it is not exposed by default, or allow
		// users to bind it to a different port. This was a quick
		// and easy way to get pprof, but it may not be the best
		// way.
		Handler: nil,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1256
1257
	}

1258
1259
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
1260
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
1261
1262
	go func() {
		<-signals
1263
		srvr.Close()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1264
		schedDone()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1265
		sched.unloadAllRunners()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1266
		done()
1267
1268
	}()

1269
1270
1271
1272
	// Locate and log what runners are present at startup
	var runnerNames []string
	for v := range runners.GetAvailableServers() {
		runnerNames = append(runnerNames, v)
1273
	}
1274
1275
	slog.Info("Dynamic LLM libraries", "runners", runnerNames)
	slog.Debug("Override detection logic by setting OLLAMA_LLM_LIBRARY")
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1276

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1277
	s.sched.Run(schedCtx)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1278
1279
1280

	// At startup we retrieve GPU information so we can get log messages before loading a model
	// This will log warnings to the log in case we have problems with detected GPUs
1281
	gpus := discover.GetGPUInfo()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1282
	gpus.LogDetails()
1283

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1284
1285
1286
1287
1288
1289
1290
	err = srvr.Serve(ln)
	// If server is closed from the signal handler, wait for the ctx to be done
	// otherwise error out quickly
	if !errors.Is(err, http.ErrServerClosed) {
		return err
	}
	<-ctx.Done()
1291
	return nil
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1292
}
Michael Yang's avatar
Michael Yang committed
1293

1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
Josh's avatar
Josh committed
1304
1305
1306
1307
			status, ok := r["status"].(int)
			if !ok {
				status = http.StatusInternalServerError
			}
1308
			if errorMsg, ok := r["error"].(string); ok {
Josh's avatar
Josh committed
1309
				c.JSON(status, gin.H{"error": errorMsg})
1310
1311
				return
			} else {
Josh's avatar
Josh committed
1312
				c.JSON(status, gin.H{"error": "unexpected error format in progress response"})
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
1323
func streamResponse(c *gin.Context, ch chan any) {
1324
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
1325
1326
1327
1328
1329
1330
1331
1332
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
1333
			slog.Info(fmt.Sprintf("streamResponse: json.Marshal failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1334
1335
1336
			return false
		}

1337
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
1338
1339
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
1340
			slog.Info(fmt.Sprintf("streamResponse: w.Write failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1341
1342
1343
1344
1345
1346
			return false
		}

		return true
	})
}
Bruce MacDonald's avatar
Bruce MacDonald committed
1347

1348
func (s *Server) PsHandler(c *gin.Context) {
1349
	models := []api.ProcessModelResponse{}
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360

	for _, v := range s.sched.loaded {
		model := v.model
		modelDetails := api.ModelDetails{
			Format:            model.Config.ModelFormat,
			Family:            model.Config.ModelFamily,
			Families:          model.Config.ModelFamilies,
			ParameterSize:     model.Config.ModelType,
			QuantizationLevel: model.Config.FileType,
		}

1361
		mr := api.ProcessModelResponse{
1362
1363
1364
1365
1366
1367
1368
1369
			Model:     model.ShortName,
			Name:      model.ShortName,
			Size:      int64(v.estimatedTotal),
			SizeVRAM:  int64(v.estimatedVRAM),
			Digest:    model.Digest,
			Details:   modelDetails,
			ExpiresAt: v.expiresAt,
		}
1370
1371
1372
1373
1374
1375
1376
1377
		// The scheduler waits to set expiresAt, so if a model is loading it's
		// possible that it will be set to the unix epoch. For those cases, just
		// calculate the time w/ the sessionDuration instead.
		var epoch time.Time
		if v.expiresAt == epoch {
			mr.ExpiresAt = time.Now().Add(v.sessionDuration)
		}

1378
1379
1380
		models = append(models, mr)
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1381
1382
1383
1384
1385
	slices.SortStableFunc(models, func(i, j api.ProcessModelResponse) int {
		// longest duration remaining listed first
		return cmp.Compare(j.ExpiresAt.Unix(), i.ExpiresAt.Unix())
	})

1386
	c.JSON(http.StatusOK, api.ProcessResponse{Models: models})
1387
1388
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1389
func (s *Server) ChatHandler(c *gin.Context) {
1390
1391
	checkpointStart := time.Now()

Bruce MacDonald's avatar
Bruce MacDonald committed
1392
	var req api.ChatRequest
Michael Yang's avatar
Michael Yang committed
1393
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1394
1395
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
1396
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1397
1398
1399
1400
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
	// expire the runner
	if len(req.Messages) == 0 && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
		model, err := GetModel(req.Model)
		if err != nil {
			switch {
			case os.IsNotExist(err):
				c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
			case err.Error() == "invalid model name":
				c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			}
			return
		}
		s.sched.expireRunner(model)

		c.JSON(http.StatusOK, api.ChatResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

Michael Yang's avatar
Michael Yang committed
1427
	caps := []Capability{CapabilityCompletion}
1428
	if len(req.Tools) > 0 {
Michael Yang's avatar
tools  
Michael Yang committed
1429
1430
1431
		caps = append(caps, CapabilityTools)
	}

Michael Yang's avatar
Michael Yang committed
1432
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
1433
1434
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support chat", req.Model)})
Bruce MacDonald's avatar
Bruce MacDonald committed
1435
		return
Michael Yang's avatar
Michael Yang committed
1436
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
1437
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
1438
1439
		return
	}
Michael Yang's avatar
Michael Yang committed
1440

1441
1442
	checkpointLoaded := time.Now()

Michael Yang's avatar
Michael Yang committed
1443
1444
	if len(req.Messages) == 0 {
		c.JSON(http.StatusOK, api.ChatResponse{
1445
			Model:      req.Model,
Michael Yang's avatar
Michael Yang committed
1446
1447
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
1448
1449
			Done:       true,
			DoneReason: "load",
Michael Yang's avatar
Michael Yang committed
1450
		})
1451
1452
1453
		return
	}

Michael Yang's avatar
Michael Yang committed
1454
	msgs := append(m.Messages, req.Messages...)
1455
	if req.Messages[0].Role != "system" && m.System != "" {
Michael Yang's avatar
Michael Yang committed
1456
		msgs = append([]api.Message{{Role: "system", Content: m.System}}, msgs...)
1457
1458
	}

Michael Yang's avatar
Michael Yang committed
1459
	prompt, images, err := chatPrompt(c.Request.Context(), m, r.Tokenize, opts, msgs, req.Tools)
Michael Yang's avatar
Michael Yang committed
1460
	if err != nil {
1461
		slog.Error("chat prompt error", "error", err)
Michael Yang's avatar
Michael Yang committed
1462
1463
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
1464
1465
	}

Michael Yang's avatar
Michael Yang committed
1466
	slog.Debug("chat request", "images", len(images), "prompt", prompt)
1467

Bruce MacDonald's avatar
Bruce MacDonald committed
1468
1469
1470
	ch := make(chan any)
	go func() {
		defer close(ch)
1471
		var sb strings.Builder
1472
		var toolCallIndex int = 0
Michael Yang's avatar
Michael Yang committed
1473
		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
Michael Yang's avatar
Michael Yang committed
1474
1475
1476
			Prompt:  prompt,
			Images:  images,
			Format:  req.Format,
Michael Yang's avatar
Michael Yang committed
1477
			Options: opts,
Michael Yang's avatar
Michael Yang committed
1478
		}, func(r llm.CompletionResponse) {
1479
			res := api.ChatResponse{
1480
1481
1482
1483
1484
				Model:      req.Model,
				CreatedAt:  time.Now().UTC(),
				Message:    api.Message{Role: "assistant", Content: r.Content},
				Done:       r.Done,
				DoneReason: r.DoneReason,
Bruce MacDonald's avatar
Bruce MacDonald committed
1485
1486
1487
1488
1489
1490
1491
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
			}
1492
1493
1494
1495
1496
1497

			if r.Done {
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
			}

1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
			// TODO: tool call checking and filtering should be moved outside of this callback once streaming
			// however this was a simple change for now without reworking streaming logic of this (and other)
			// handlers
			if req.Stream != nil && !*req.Stream || len(req.Tools) == 0 {
				ch <- res
				return
			}

			// Streaming tool calls:
			// If tools are recognized, use a flag to track the sending of a tool downstream
			// This ensures that content is cleared from the message on the last chunk sent
			sb.WriteString(r.Content)
			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
				res.Message.ToolCalls = toolCalls
1512
1513
1514
1515
				for i := range toolCalls {
					toolCalls[i].Function.Index = toolCallIndex
					toolCallIndex++
				}
1516
1517
1518
1519
1520
1521
1522
1523
				res.Message.Content = ""
				sb.Reset()
				ch <- res
				return
			}

			if r.Done {
				// Send any remaining content if no tool calls were detected
1524
				if toolCallIndex == 0 {
1525
1526
1527
1528
					res.Message.Content = sb.String()
				}
				ch <- res
			}
Michael Yang's avatar
Michael Yang committed
1529
		}); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1530
1531
1532
1533
1534
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
tools  
Michael Yang committed
1535
		var resp api.ChatResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
1536
		var sb strings.Builder
Michael Yang's avatar
Michael Yang committed
1537
1538
		for rr := range ch {
			switch t := rr.(type) {
1539
			case api.ChatResponse:
Michael Yang's avatar
Michael Yang committed
1540
				sb.WriteString(t.Message.Content)
Michael Yang's avatar
tools  
Michael Yang committed
1541
				resp = t
1542
			case gin.H:
Michael Yang's avatar
Michael Yang committed
1543
1544
1545
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
1546
				}
Michael Yang's avatar
Michael Yang committed
1547
1548
1549

				c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
				return
1550
			default:
Michael Yang's avatar
Michael Yang committed
1551
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
1552
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
1553
1554
			}
		}
1555

Michael Yang's avatar
tools  
Michael Yang committed
1556
		resp.Message.Content = sb.String()
1557
1558
1559
1560
1561
1562

		if len(req.Tools) > 0 {
			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
				resp.Message.ToolCalls = toolCalls
				resp.Message.Content = ""
			}
Michael Yang's avatar
tools  
Michael Yang committed
1563
1564
1565
		}

		c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
1566
1567
1568
1569
1570
		return
	}

	streamResponse(c, ch)
}
1571

Michael Yang's avatar
Michael Yang committed
1572
func handleScheduleError(c *gin.Context, name string, err error) {
Michael Yang's avatar
Michael Yang committed
1573
	switch {
1574
	case errors.Is(err, errCapabilities), errors.Is(err, errRequired):
Michael Yang's avatar
Michael Yang committed
1575
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
1576
	case errors.Is(err, context.Canceled):
1577
		c.JSON(499, gin.H{"error": "request canceled"})
Michael Yang's avatar
Michael Yang committed
1578
	case errors.Is(err, ErrMaxQueue):
1579
		c.JSON(http.StatusServiceUnavailable, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
1580
1581
	case errors.Is(err, os.ErrNotExist):
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found, try pulling it first", name)})
Michael Yang's avatar
Michael Yang committed
1582
1583
	default:
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
1584
1585
	}
}