routes.go 39.8 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
Michael Yang's avatar
Michael Yang committed
4
	"bytes"
Michael Yang's avatar
Michael Yang committed
5
	"cmp"
6
	"context"
7
	"encoding/binary"
Michael Yang's avatar
Michael Yang committed
8
	"encoding/json"
9
	"errors"
10
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
11
	"io"
12
	"log/slog"
13
	"math"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
14
15
	"net"
	"net/http"
16
	"net/netip"
17
	"os"
18
	"os/signal"
Michael Yang's avatar
Michael Yang committed
19
	"path/filepath"
20
	"slices"
Michael Yang's avatar
Michael Yang committed
21
	"strings"
22
	"syscall"
23
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
24

Michael Yang's avatar
Michael Yang committed
25
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
26
	"github.com/gin-gonic/gin"
27
	"golang.org/x/sync/errgroup"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
28

29
	"github.com/ollama/ollama/api"
30
	"github.com/ollama/ollama/discover"
31
	"github.com/ollama/ollama/envconfig"
32
33
	"github.com/ollama/ollama/llm"
	"github.com/ollama/ollama/openai"
34
	"github.com/ollama/ollama/parser"
35
	"github.com/ollama/ollama/runners"
36
	"github.com/ollama/ollama/server/imageproc"
Michael Yang's avatar
Michael Yang committed
37
	"github.com/ollama/ollama/template"
38
	"github.com/ollama/ollama/types/errtypes"
Michael Yang's avatar
Michael Yang committed
39
	"github.com/ollama/ollama/types/model"
40
	"github.com/ollama/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
41
42
)

Michael Yang's avatar
Michael Yang committed
43
44
var mode string = gin.DebugMode

45
type Server struct {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
46
47
	addr  net.Addr
	sched *Scheduler
48
49
}

Michael Yang's avatar
Michael Yang committed
50
51
52
53
54
55
56
57
58
59
60
61
func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Michael Yang's avatar
lint  
Michael Yang committed
62
63
64
65
var (
	errRequired    = errors.New("is required")
	errBadTemplate = errors.New("template error")
)
Michael Yang's avatar
Michael Yang committed
66

67
68
69
70
71
72
73
74
75
76
77
func modelOptions(model *Model, requestOpts map[string]interface{}) (api.Options, error) {
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		return api.Options{}, err
	}

	if err := opts.FromMap(requestOpts); err != nil {
		return api.Options{}, err
	}

	return opts, nil
Bruce MacDonald's avatar
Bruce MacDonald committed
78
79
}

Michael Yang's avatar
Michael Yang committed
80
81
82
// scheduleRunner schedules a runner after validating inputs such as capabilities and model options.
// It returns the allocated runner, model instance, and consolidated options if successful and error otherwise.
func (s *Server) scheduleRunner(ctx context.Context, name string, caps []Capability, requestOpts map[string]any, keepAlive *api.Duration) (llm.LlamaServer, *Model, *api.Options, error) {
Michael Yang's avatar
Michael Yang committed
83
	if name == "" {
Michael Yang's avatar
Michael Yang committed
84
		return nil, nil, nil, fmt.Errorf("model %w", errRequired)
Bruce MacDonald's avatar
Bruce MacDonald committed
85
86
	}

Michael Yang's avatar
Michael Yang committed
87
	model, err := GetModel(name)
Bruce MacDonald's avatar
Bruce MacDonald committed
88
	if err != nil {
Michael Yang's avatar
Michael Yang committed
89
		return nil, nil, nil, err
90
91
	}

Michael Yang's avatar
Michael Yang committed
92
	if err := model.CheckCapabilities(caps...); err != nil {
Michael Yang's avatar
Michael Yang committed
93
		return nil, nil, nil, fmt.Errorf("%s %w", name, err)
94
95
	}

Michael Yang's avatar
Michael Yang committed
96
	opts, err := modelOptions(model, requestOpts)
97
	if err != nil {
Michael Yang's avatar
Michael Yang committed
98
		return nil, nil, nil, err
99
100
	}

Michael Yang's avatar
Michael Yang committed
101
	runnerCh, errCh := s.sched.GetRunner(ctx, model, opts, keepAlive)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
102
103
	var runner *runnerRef
	select {
Michael Yang's avatar
Michael Yang committed
104
105
	case runner = <-runnerCh:
	case err = <-errCh:
Michael Yang's avatar
Michael Yang committed
106
		return nil, nil, nil, err
Bruce MacDonald's avatar
Bruce MacDonald committed
107
108
	}

Michael Yang's avatar
Michael Yang committed
109
	return runner.llama, model, &opts, nil
Michael Yang's avatar
Michael Yang committed
110
111
112
}

func (s *Server) GenerateHandler(c *gin.Context) {
113
	checkpointStart := time.Now()
Michael Yang's avatar
Michael Yang committed
114
115
116
117
118
119
	var req api.GenerateRequest
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
120
121
122
		return
	}

123
124
125
126
127
128
129
130
131
132
133
134
135
	model, err := GetModel(req.Model)
	if err != nil {
		switch {
		case os.IsNotExist(err):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		case err.Error() == "invalid model name":
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

Patrick Devine's avatar
Patrick Devine committed
136
137
138
139
140
141
142
143
144
145
146
147
148
149
	// expire the runner
	if req.Prompt == "" && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
		s.sched.expireRunner(model)

		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Response:   "",
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

150
	if req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0) {
Michael Yang's avatar
Michael Yang committed
151
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
Michael Yang's avatar
Michael Yang committed
152
153
154
		return
	}

Michael Yang's avatar
Michael Yang committed
155
	caps := []Capability{CapabilityCompletion}
156
157
158
159
	if req.Suffix != "" {
		caps = append(caps, CapabilityInsert)
	}

Michael Yang's avatar
Michael Yang committed
160
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
161
162
163
164
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support generate", req.Model)})
		return
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
165
166
167
168
		handleScheduleError(c, req.Model, err)
		return
	}

169
170
	checkpointLoaded := time.Now()

171
	// load the model
Michael Yang's avatar
Michael Yang committed
172
173
174
175
176
177
178
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Done:       true,
			DoneReason: "load",
		})
Michael Yang's avatar
Michael Yang committed
179
180
		return
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
181

182
183
184
185
186
187
	isMllama := checkMllamaModelFamily(model)
	if isMllama && len(req.Images) > 1 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "this model only supports one image: more than one image sent"})
		return
	}

Michael Yang's avatar
Michael Yang committed
188
189
	images := make([]llm.ImageData, len(req.Images))
	for i := range req.Images {
190
191
192
193
194
195
196
197
198
199
200
201
202
203
		if isMllama {
			data, aspectRatioID, err := imageproc.Preprocess(req.Images[i])
			if err != nil {
				c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "error processing image"})
				return
			}

			buf := new(bytes.Buffer)
			err = binary.Write(buf, binary.LittleEndian, data)
			if err != nil {
				c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "error processing image"})
				return
			}

204
			images[i] = llm.ImageData{ID: i, Data: buf.Bytes(), AspectRatioID: aspectRatioID}
205
206
207
		} else {
			images[i] = llm.ImageData{ID: i, Data: req.Images[i]}
		}
Michael Yang's avatar
Michael Yang committed
208
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
209

Michael Yang's avatar
Michael Yang committed
210
211
	prompt := req.Prompt
	if !req.Raw {
Michael Yang's avatar
Michael Yang committed
212
		tmpl := m.Template
Michael Yang's avatar
Michael Yang committed
213
214
215
216
217
218
219
220
		if req.Template != "" {
			tmpl, err = template.Parse(req.Template)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

221
222
223
224
225
226
227
228
229
230
231
232
		var values template.Values
		if req.Suffix != "" {
			values.Prompt = prompt
			values.Suffix = req.Suffix
		} else {
			var msgs []api.Message
			if req.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: req.System})
			} else if m.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: m.System})
			}

Michael Yang's avatar
Michael Yang committed
233
234
235
236
			if req.Context == nil {
				msgs = append(msgs, m.Messages...)
			}

237
			for _, i := range images {
238
				imgPrompt := ""
239
				if isMllama {
240
					imgPrompt = "<|image|>"
241
				}
242
				msgs = append(msgs, api.Message{Role: "user", Content: fmt.Sprintf("[img-%d]"+imgPrompt, i.ID)})
243
244
245
246
247
			}

			values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt})
		}

Michael Yang's avatar
Michael Yang committed
248
249
		var b bytes.Buffer
		if req.Context != nil {
250
			slog.Warn("the context field is deprecated and will be removed in a future version of Ollama")
251
			s, err := r.Detokenize(c.Request.Context(), req.Context)
Michael Yang's avatar
Michael Yang committed
252
253
254
255
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
256
			b.WriteString(s)
Michael Yang's avatar
Michael Yang committed
257
		}
258
259
260
261
262
263
264

		if err := tmpl.Execute(&b, values); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		prompt = b.String()
Bruce MacDonald's avatar
Bruce MacDonald committed
265
266
	}

267
	slog.Debug("generate request", "images", len(images), "prompt", prompt)
268

Bruce MacDonald's avatar
Bruce MacDonald committed
269
270
	ch := make(chan any)
	go func() {
271
272
		// TODO (jmorganca): avoid building the response twice both here and below
		var sb strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
273
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
274
		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
Michael Yang's avatar
Michael Yang committed
275
276
			Prompt:  prompt,
			Images:  images,
277
			Format:  req.Format,
Michael Yang's avatar
Michael Yang committed
278
			Options: opts,
279
280
		}, func(cr llm.CompletionResponse) {
			res := api.GenerateResponse{
281
282
				Model:      req.Model,
				CreatedAt:  time.Now().UTC(),
283
284
285
				Response:   cr.Content,
				Done:       cr.Done,
				DoneReason: cr.DoneReason,
Bruce MacDonald's avatar
Bruce MacDonald committed
286
				Metrics: api.Metrics{
287
288
289
290
					PromptEvalCount:    cr.PromptEvalCount,
					PromptEvalDuration: cr.PromptEvalDuration,
					EvalCount:          cr.EvalCount,
					EvalDuration:       cr.EvalDuration,
Bruce MacDonald's avatar
Bruce MacDonald committed
291
				},
Bruce MacDonald's avatar
Bruce MacDonald committed
292
			}
293
294
295
296
297
298
299
300
301
302

			if _, err := sb.WriteString(cr.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
			}

			if cr.Done {
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)

				if !req.Raw {
303
					tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String())
304
305
306
307
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
308
					res.Context = tokens
309
310
311
312
				}
			}

			ch <- res
Michael Yang's avatar
Michael Yang committed
313
		}); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
314
315
316
317
318
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
Michael Yang committed
319
		var r api.GenerateResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
320
		var sb strings.Builder
Michael Yang's avatar
Michael Yang committed
321
322
		for rr := range ch {
			switch t := rr.(type) {
323
			case api.GenerateResponse:
Michael Yang's avatar
Michael Yang committed
324
325
				sb.WriteString(t.Response)
				r = t
326
			case gin.H:
Michael Yang's avatar
Michael Yang committed
327
328
329
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
330
				}
Michael Yang's avatar
Michael Yang committed
331
332
333

				c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
				return
334
			default:
Michael Yang's avatar
Michael Yang committed
335
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
Bruce MacDonald's avatar
Bruce MacDonald committed
336
337
338
				return
			}
		}
339

Michael Yang's avatar
Michael Yang committed
340
341
		r.Response = sb.String()
		c.JSON(http.StatusOK, r)
Bruce MacDonald's avatar
Bruce MacDonald committed
342
343
344
345
346
347
		return
	}

	streamResponse(c, ch)
}

348
func (s *Server) EmbedHandler(c *gin.Context) {
349
	checkpointStart := time.Now()
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
	var req api.EmbedRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	truncate := true

	if req.Truncate != nil && !*req.Truncate {
		truncate = false
	}

	var input []string

	switch i := req.Input.(type) {
	case string:
		if len(i) > 0 {
			input = append(input, i)
		}
	case []any:
		for _, v := range i {
			if _, ok := v.(string); !ok {
				c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
				return
			}
			input = append(input, v.(string))
		}
	default:
383
384
385
386
		if req.Input != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
			return
		}
387
388
389
390
391
392
393
394
	}

	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}

395
396
	checkpointLoaded := time.Now()

397
398
399
400
401
	if len(input) == 0 {
		c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
		return
	}

402
403
404
405
406
407
	kvData, err := getKVData(m.ModelPath, false)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

408
	var count int
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
	for i, s := range input {
		tokens, err := r.Tokenize(c.Request.Context(), s)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		ctxLen := min(opts.NumCtx, int(kvData.ContextLength()))
		if len(tokens) > ctxLen {
			if !truncate {
				c.JSON(http.StatusBadRequest, gin.H{"error": "input length exceeds maximum context length"})
				return
			}

			tokens = tokens[:ctxLen]
			s, err = r.Detokenize(c.Request.Context(), tokens)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

431
432
		count += len(tokens)

433
434
		input[i] = s
	}
435
436
437
438
439
440
441
442
443
444
445
446

	var g errgroup.Group
	embeddings := make([][]float32, len(input))
	for i, text := range input {
		g.Go(func() error {
			embedding, err := r.Embedding(c.Request.Context(), text)
			if err != nil {
				return err
			}
			embeddings[i] = normalize(embedding)
			return nil
		})
447
448
	}

449
450
451
452
	if err := g.Wait(); err != nil {
		slog.Error("embedding generation failed", "error", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embeddings: %v", err)})
		return
453
454
455
	}

	resp := api.EmbedResponse{
456
		Model:           req.Model,
457
		Embeddings:      embeddings,
458
459
		TotalDuration:   time.Since(checkpointStart),
		LoadDuration:    checkpointLoaded.Sub(checkpointStart),
460
		PromptEvalCount: count,
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
	}
	c.JSON(http.StatusOK, resp)
}

func normalize(vec []float32) []float32 {
	var sum float32
	for _, v := range vec {
		sum += v * v
	}

	norm := float32(0.0)
	if sum > 0 {
		norm = float32(1.0 / math.Sqrt(float64(sum)))
	}

	for i := range vec {
		vec[i] *= norm
	}
	return vec
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
482
func (s *Server) EmbeddingsHandler(c *gin.Context) {
Bruce MacDonald's avatar
Bruce MacDonald committed
483
	var req api.EmbeddingRequest
Michael Yang's avatar
Michael Yang committed
484
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
485
486
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
487
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
488
489
490
491
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
492
	r, _, _, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
Bruce MacDonald's avatar
Bruce MacDonald committed
493
	if err != nil {
Michael Yang's avatar
Michael Yang committed
494
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
495
496
497
		return
	}

498
499
500
	// an empty request loads the model
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float64{}})
Bruce MacDonald's avatar
Bruce MacDonald committed
501
502
503
		return
	}

504
	embedding, err := r.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
505
	if err != nil {
506
		slog.Info(fmt.Sprintf("embedding generation failed: %v", err))
507
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embedding: %v", err)})
Bruce MacDonald's avatar
Bruce MacDonald committed
508
509
510
		return
	}

511
512
513
	var e []float64
	for _, v := range embedding {
		e = append(e, float64(v))
514
515
516
	}

	resp := api.EmbeddingResponse{
517
		Embedding: e,
518
519
	}
	c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
520
521
}

522
func (s *Server) PullHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
523
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
524
525
526
527
528
529
530
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
531
532
533
		return
	}

534
535
536
537
538
539
	name := model.ParseName(cmp.Or(req.Model, req.Name))
	if !name.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid model name"})
		return
	}

540
541
	name, err = getExistingName(name)
	if err != nil {
542
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
543
544
545
		return
	}

546
547
548
	ch := make(chan any)
	go func() {
		defer close(ch)
549
550
		fn := func(r api.ProgressResponse) {
			ch <- r
551
		}
552

Michael Yang's avatar
Michael Yang committed
553
		regOpts := &registryOptions{
554
555
556
			Insecure: req.Insecure,
		}

557
558
559
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

560
		if err := PullModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
561
			ch <- gin.H{"error": err.Error()}
562
563
564
		}
	}()

565
566
567
568
569
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

570
571
572
	streamResponse(c, ch)
}

573
func (s *Server) PushHandler(c *gin.Context) {
574
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
575
576
577
578
579
580
581
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
582
583
		return
	}
Michael Yang's avatar
Michael Yang committed
584

Michael Yang's avatar
Michael Yang committed
585
586
587
588
589
590
591
	var model string
	if req.Model != "" {
		model = req.Model
	} else if req.Name != "" {
		model = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
592
593
594
		return
	}

595
596
597
	ch := make(chan any)
	go func() {
		defer close(ch)
598
599
		fn := func(r api.ProgressResponse) {
			ch <- r
600
		}
601

Michael Yang's avatar
Michael Yang committed
602
		regOpts := &registryOptions{
603
604
605
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
606
607
608
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

Michael Yang's avatar
Michael Yang committed
609
		if err := PushModel(ctx, model, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
610
			ch <- gin.H{"error": err.Error()}
611
612
613
		}
	}()

614
615
616
617
618
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

619
620
621
	streamResponse(c, ch)
}

622
623
624
625
626
// getExistingName returns the original, on disk name if the input name is a
// case-insensitive match, otherwise it returns the input name.
func getExistingName(n model.Name) (model.Name, error) {
	var zero model.Name
	existing, err := Manifests(true)
627
	if err != nil {
628
		return zero, err
629
	}
630
631
632
	for e := range existing {
		if n.EqualFold(e) {
			return e, nil
633
634
		}
	}
635
	return n, nil
636
637
}

638
func (s *Server) CreateHandler(c *gin.Context) {
639
640
	var r api.CreateRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
641
642
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
643
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
644
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
645
		return
646
647
	}

648
	name := model.ParseName(cmp.Or(r.Model, r.Name))
Michael Yang's avatar
Michael Yang committed
649
	if !name.IsValid() {
650
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": errtypes.InvalidModelNameErrMsg})
651
652
653
		return
	}

654
655
	name, err := getExistingName(name)
	if err != nil {
656
657
658
659
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

660
	if r.Path == "" && r.Modelfile == "" {
Michael Yang's avatar
Michael Yang committed
661
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"})
Michael Yang's avatar
Michael Yang committed
662
663
		return
	}
Michael Yang's avatar
Michael Yang committed
664

665
666
667
	var sr io.Reader = strings.NewReader(r.Modelfile)
	if r.Path != "" && r.Modelfile == "" {
		f, err := os.Open(r.Path)
Michael Yang's avatar
Michael Yang committed
668
669
670
671
		if err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)})
			return
		}
Michael Yang's avatar
Michael Yang committed
672
		defer f.Close()
Michael Yang's avatar
Michael Yang committed
673

674
		sr = f
Michael Yang's avatar
Michael Yang committed
675
	}
Michael Yang's avatar
Michael Yang committed
676

677
	f, err := parser.ParseFile(sr)
Michael Yang's avatar
Michael Yang committed
678
679
680
681
682
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
683
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
684
685
	go func() {
		defer close(ch)
686
687
		fn := func(resp api.ProgressResponse) {
			ch <- resp
688
689
		}

690
691
692
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

693
		quantization := cmp.Or(r.Quantize, r.Quantization)
Josh's avatar
Josh committed
694
695
696
		if err := CreateModel(ctx, name, filepath.Dir(r.Path), strings.ToUpper(quantization), f, fn); errors.Is(err, errBadTemplate) {
			ch <- gin.H{"error": err.Error(), "status": http.StatusBadRequest}
		} else if err != nil {
Michael Yang's avatar
Michael Yang committed
697
			ch <- gin.H{"error": err.Error()}
698
		}
Michael Yang's avatar
Michael Yang committed
699
	}()
Michael Yang's avatar
Michael Yang committed
700

701
	if r.Stream != nil && !*r.Stream {
702
703
704
705
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
706
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
707
708
}

709
func (s *Server) DeleteHandler(c *gin.Context) {
710
711
	var r api.DeleteRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
712
713
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
714
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
715
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
716
717
718
		return
	}

719
720
721
	n := model.ParseName(cmp.Or(r.Model, r.Name))
	if !n.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("name %q is invalid", cmp.Or(r.Model, r.Name))})
722
723
		return
	}
Michael Yang's avatar
Michael Yang committed
724

725
	m, err := ParseNamedManifest(n)
Michael Yang's avatar
Michael Yang committed
726
	if err != nil {
727
728
729
730
731
732
		switch {
		case os.IsNotExist(err):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", cmp.Or(r.Model, r.Name))})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
Michael Yang's avatar
Michael Yang committed
733
734
735
		return
	}

736
	if err := m.Remove(); err != nil {
Michael Yang's avatar
Michael Yang committed
737
738
739
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
740
741
742
743
744

	if err := m.RemoveLayers(); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
745
746
}

747
func (s *Server) ShowHandler(c *gin.Context) {
Patrick Devine's avatar
Patrick Devine committed
748
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
749
750
751
752
753
754
755
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
756
757
758
		return
	}

Michael Yang's avatar
Michael Yang committed
759
	if req.Model != "" {
Michael Yang's avatar
Michael Yang committed
760
		// noop
Michael Yang's avatar
Michael Yang committed
761
	} else if req.Name != "" {
Michael Yang's avatar
Michael Yang committed
762
		req.Model = req.Name
Michael Yang's avatar
Michael Yang committed
763
	} else {
764
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
765
766
767
		return
	}

768
	resp, err := GetModelInfo(req)
Patrick Devine's avatar
Patrick Devine committed
769
	if err != nil {
770
771
		switch {
		case os.IsNotExist(err):
Michael Yang's avatar
Michael Yang committed
772
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
773
774
775
		case err.Error() == "invalid model name":
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
Patrick Devine's avatar
Patrick Devine committed
776
777
778
779
780
781
782
783
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

784
func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
785
	m, err := GetModel(req.Model)
Patrick Devine's avatar
Patrick Devine committed
786
787
788
789
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
790
	modelDetails := api.ModelDetails{
791
792
793
794
795
796
		ParentModel:       m.ParentModel,
		Format:            m.Config.ModelFormat,
		Family:            m.Config.ModelFamily,
		Families:          m.Config.ModelFamilies,
		ParameterSize:     m.Config.ModelType,
		QuantizationLevel: m.Config.FileType,
Patrick Devine's avatar
Patrick Devine committed
797
798
	}

799
	if req.System != "" {
800
		m.System = req.System
801
802
	}

Michael Yang's avatar
Michael Yang committed
803
804
805
	msgs := make([]api.Message, len(m.Messages))
	for i, msg := range m.Messages {
		msgs[i] = api.Message{Role: msg.Role, Content: msg.Content}
806
807
	}

808
809
	n := model.ParseName(req.Model)
	if !n.IsValid() {
Michael Yang's avatar
lint  
Michael Yang committed
810
		return nil, errors.New("invalid model name")
811
812
813
814
815
816
817
	}

	manifest, err := ParseNamedManifest(n)
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
818
	resp := &api.ShowResponse{
819
820
		License:    strings.Join(m.License, "\n"),
		System:     m.System,
Michael Yang's avatar
Michael Yang committed
821
		Template:   m.Template.String(),
822
823
824
		Details:    modelDetails,
		Messages:   msgs,
		ModifiedAt: manifest.fi.ModTime(),
Patrick Devine's avatar
Patrick Devine committed
825
826
827
828
	}

	var params []string
	cs := 30
829
	for k, v := range m.Options {
Patrick Devine's avatar
Patrick Devine committed
830
831
832
		switch val := v.(type) {
		case []interface{}:
			for _, nv := range val {
Patrick Devine's avatar
Patrick Devine committed
833
				params = append(params, fmt.Sprintf("%-*s %#v", cs, k, nv))
Patrick Devine's avatar
Patrick Devine committed
834
			}
Patrick Devine's avatar
Patrick Devine committed
835
836
		default:
			params = append(params, fmt.Sprintf("%-*s %#v", cs, k, v))
Patrick Devine's avatar
Patrick Devine committed
837
838
839
840
		}
	}
	resp.Parameters = strings.Join(params, "\n")

841
842
	for k, v := range req.Options {
		if _, ok := req.Options[k]; ok {
843
			m.Options[k] = v
844
845
846
		}
	}

847
	var sb strings.Builder
848
	fmt.Fprintln(&sb, "# Modelfile generated by \"ollama show\"")
849
	fmt.Fprintln(&sb, "# To build a new Modelfile based on this, replace FROM with:")
850
851
	fmt.Fprintf(&sb, "# FROM %s\n\n", m.ShortName)
	fmt.Fprint(&sb, m.String())
852
	resp.Modelfile = sb.String()
853

854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
	kvData, err := getKVData(m.ModelPath, req.Verbose)
	if err != nil {
		return nil, err
	}
	delete(kvData, "general.name")
	delete(kvData, "tokenizer.chat_template")
	resp.ModelInfo = kvData

	if len(m.ProjectorPaths) > 0 {
		projectorData, err := getKVData(m.ProjectorPaths[0], req.Verbose)
		if err != nil {
			return nil, err
		}
		resp.ProjectorInfo = projectorData
	}

Patrick Devine's avatar
Patrick Devine committed
870
871
872
	return resp, nil
}

873
func getKVData(digest string, verbose bool) (llm.KV, error) {
874
875
876
877
878
	maxArraySize := 0
	if verbose {
		maxArraySize = -1
	}
	kvData, err := llm.LoadModel(digest, maxArraySize)
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
	if err != nil {
		return nil, err
	}

	kv := kvData.KV()

	if !verbose {
		for k := range kv {
			if t, ok := kv[k].([]any); len(t) > 5 && ok {
				kv[k] = []any{}
			}
		}
	}

	return kv, nil
}

896
func (s *Server) ListHandler(c *gin.Context) {
897
	ms, err := Manifests(true)
Patrick Devine's avatar
Patrick Devine committed
898
899
900
901
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
902

903
	models := []api.ListModelResponse{}
904
905
	for n, m := range ms {
		var cf ConfigV2
906
907
908
909
910
911
912
913
914
915
916
917
918

		if m.Config.Digest != "" {
			f, err := m.Config.Open()
			if err != nil {
				slog.Warn("bad manifest filepath", "name", n, "error", err)
				continue
			}
			defer f.Close()

			if err := json.NewDecoder(f).Decode(&cf); err != nil {
				slog.Warn("bad manifest config", "name", n, "error", err)
				continue
			}
Patrick Devine's avatar
Patrick Devine committed
919
		}
Michael Yang's avatar
Michael Yang committed
920

921
		// tag should never be masked
922
		models = append(models, api.ListModelResponse{
923
924
925
926
927
928
929
930
931
932
933
934
935
			Model:      n.DisplayShortest(),
			Name:       n.DisplayShortest(),
			Size:       m.Size(),
			Digest:     m.digest,
			ModifiedAt: m.fi.ModTime(),
			Details: api.ModelDetails{
				Format:            cf.ModelFormat,
				Family:            cf.ModelFamily,
				Families:          cf.ModelFamilies,
				ParameterSize:     cf.ModelType,
				QuantizationLevel: cf.FileType,
			},
		})
Patrick Devine's avatar
Patrick Devine committed
936
937
	}

938
	slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
939
940
941
942
		// most recently modified first
		return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
	})

Michael Yang's avatar
Michael Yang committed
943
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
944
945
}

946
func (s *Server) CopyHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
947
948
	var r api.CopyRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
949
950
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
951
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
952
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
953
954
955
		return
	}

Michael Yang's avatar
Michael Yang committed
956
957
	src := model.ParseName(r.Source)
	if !src.IsValid() {
Michael Yang's avatar
Michael Yang committed
958
959
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("source %q is invalid", r.Source)})
		return
960
	}
961
962
963
964
965
	src, err := getExistingName(src)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
966

Michael Yang's avatar
Michael Yang committed
967
968
	dst := model.ParseName(r.Destination)
	if !dst.IsValid() {
969
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("destination %q is invalid", r.Destination)})
Patrick Devine's avatar
Patrick Devine committed
970
971
		return
	}
972
973
	dst, err = getExistingName(dst)
	if err != nil {
974
975
976
977
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
978
979
980
981
982
	if err := CopyModel(src, dst); errors.Is(err, os.ErrNotExist) {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
	} else if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
	}
Patrick Devine's avatar
Patrick Devine committed
983
984
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
985
func (s *Server) HeadBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
986
987
988
989
990
991
992
993
994
995
996
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); err != nil {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	}

Michael Yang's avatar
Michael Yang committed
997
	c.Status(http.StatusOK)
Michael Yang's avatar
Michael Yang committed
998
999
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1000
func (s *Server) CreateBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
1001
1002
	if ib, ok := intermediateBlobs[c.Param("digest")]; ok {
		p, err := GetBlobsPath(ib)
1003
1004
1005
1006
1007
1008
		if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
Michael Yang's avatar
Michael Yang committed
1009
1010
			slog.Info("evicting intermediate blob which no longer exists", "digest", ib)
			delete(intermediateBlobs, c.Param("digest"))
1011
1012
1013
1014
1015
1016
1017
1018
1019
		} else if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		} else {
			c.Status(http.StatusOK)
			return
		}
	}

1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	_, err = os.Stat(path)
	switch {
	case errors.Is(err, os.ErrNotExist):
		// noop
	case err != nil:
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	default:
		c.Status(http.StatusOK)
		return
	}

1038
	layer, err := NewLayer(c.Request.Body, "")
Michael Yang's avatar
Michael Yang committed
1039
1040
1041
1042
1043
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

1044
1045
	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
Michael Yang's avatar
Michael Yang committed
1046
1047
1048
		return
	}

Michael Yang's avatar
Michael Yang committed
1049
	c.Status(http.StatusCreated)
Michael Yang's avatar
Michael Yang committed
1050
1051
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
func isLocalIP(ip netip.Addr) bool {
	if interfaces, err := net.Interfaces(); err == nil {
		for _, iface := range interfaces {
			addrs, err := iface.Addrs()
			if err != nil {
				continue
			}

			for _, a := range addrs {
				if parsed, _, err := net.ParseCIDR(a.String()); err == nil {
					if parsed.String() == ip.String() {
						return true
					}
				}
			}
		}
	}

	return false
}

1073
func allowedHost(host string) bool {
1074
1075
	host = strings.ToLower(host)

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1076
	if host == "" || host == "localhost" {
1077
1078
1079
		return true
	}

1080
	if hostname, err := os.Hostname(); err == nil && host == strings.ToLower(hostname) {
1081
1082
1083
		return true
	}

Michael Yang's avatar
lint  
Michael Yang committed
1084
	tlds := []string{
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1085
1086
1087
		"localhost",
		"local",
		"internal",
1088
	}
1089

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1090
	// check if the host is a local TLD
1091
1092
1093
1094
1095
1096
	for _, tld := range tlds {
		if strings.HasSuffix(host, "."+tld) {
			return true
		}
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1097
	return false
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1098
}
1099

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1100
1101
1102
func allowedHostsMiddleware(addr net.Addr) gin.HandlerFunc {
	return func(c *gin.Context) {
		if addr == nil {
1103
1104
1105
1106
			c.Next()
			return
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1107
		if addr, err := netip.ParseAddrPort(addr.String()); err == nil && !addr.Addr().IsLoopback() {
1108
1109
1110
1111
1112
1113
1114
1115
1116
			c.Next()
			return
		}

		host, _, err := net.SplitHostPort(c.Request.Host)
		if err != nil {
			host = c.Request.Host
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1117
		if addr, err := netip.ParseAddr(host); err == nil {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1118
			if addr.IsLoopback() || addr.IsPrivate() || addr.IsUnspecified() || isLocalIP(addr) {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1119
1120
1121
1122
1123
				c.Next()
				return
			}
		}

1124
		if allowedHost(host) {
Michael Yang's avatar
lint  
Michael Yang committed
1125
			if c.Request.Method == http.MethodOptions {
1126
1127
1128
1129
				c.AbortWithStatus(http.StatusNoContent)
				return
			}

1130
1131
1132
1133
1134
1135
			c.Next()
			return
		}

		c.AbortWithStatus(http.StatusForbidden)
	}
1136
}
1137

1138
func (s *Server) GenerateRoutes() http.Handler {
Michael Yang's avatar
Michael Yang committed
1139
1140
	config := cors.DefaultConfig()
	config.AllowWildcard = true
1141
	config.AllowBrowserExtensions = true
1142
	config.AllowHeaders = []string{"Authorization", "Content-Type", "User-Agent", "Accept", "X-Requested-With"}
1143
	openAIProperties := []string{"lang", "package-version", "os", "arch", "retry-count", "runtime", "runtime-version", "async"}
royjhan's avatar
royjhan committed
1144
1145
1146
	for _, prop := range openAIProperties {
		config.AllowHeaders = append(config.AllowHeaders, "x-stainless-"+prop)
	}
Michael Yang's avatar
origins  
Michael Yang committed
1147
	config.AllowOrigins = envconfig.Origins()
Michael Yang's avatar
Michael Yang committed
1148

Bruce MacDonald's avatar
Bruce MacDonald committed
1149
	r := gin.Default()
1150
1151
	r.Use(
		cors.New(config),
1152
		allowedHostsMiddleware(s.addr),
1153
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
1154

1155
	r.POST("/api/pull", s.PullHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1156
1157
	r.POST("/api/generate", s.GenerateHandler)
	r.POST("/api/chat", s.ChatHandler)
1158
	r.POST("/api/embed", s.EmbedHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1159
	r.POST("/api/embeddings", s.EmbeddingsHandler)
1160
1161
1162
1163
1164
	r.POST("/api/create", s.CreateHandler)
	r.POST("/api/push", s.PushHandler)
	r.POST("/api/copy", s.CopyHandler)
	r.DELETE("/api/delete", s.DeleteHandler)
	r.POST("/api/show", s.ShowHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1165
1166
	r.POST("/api/blobs/:digest", s.CreateBlobHandler)
	r.HEAD("/api/blobs/:digest", s.HeadBlobHandler)
1167
	r.GET("/api/ps", s.PsHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1168

1169
	// Compatibility endpoints
1170
	r.POST("/v1/chat/completions", openai.ChatMiddleware(), s.ChatHandler)
1171
	r.POST("/v1/completions", openai.CompletionsMiddleware(), s.GenerateHandler)
1172
	r.POST("/v1/embeddings", openai.EmbeddingsMiddleware(), s.EmbedHandler)
1173
1174
	r.GET("/v1/models", openai.ListMiddleware(), s.ListHandler)
	r.GET("/v1/models/:model", openai.RetrieveMiddleware(), s.ShowHandler)
1175

Michael Yang's avatar
Michael Yang committed
1176
1177
1178
1179
1180
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

1181
		r.Handle(method, "/api/tags", s.ListHandler)
Michael Yang's avatar
Michael Yang committed
1182
1183
1184
		r.Handle(method, "/api/version", func(c *gin.Context) {
			c.JSON(http.StatusOK, gin.H{"version": version.Version})
		})
Michael Yang's avatar
Michael Yang committed
1185
1186
	}

1187
1188
1189
1190
	return r
}

func Serve(ln net.Listener) error {
Michael Yang's avatar
Michael Yang committed
1191
	level := slog.LevelInfo
Michael Yang's avatar
Michael Yang committed
1192
	if envconfig.Debug() {
Michael Yang's avatar
Michael Yang committed
1193
		level = slog.LevelDebug
1194
	}
Michael Yang's avatar
Michael Yang committed
1195

1196
	slog.Info("server config", "env", envconfig.Values())
Michael Yang's avatar
Michael Yang committed
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
	handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
		Level:     level,
		AddSource: true,
		ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr {
			if attr.Key == slog.SourceKey {
				source := attr.Value.Any().(*slog.Source)
				source.File = filepath.Base(source.File)
			}

			return attr
		},
	})

	slog.SetDefault(slog.New(handler))

1212
1213
1214
1215
1216
1217
1218
1219
	blobsDir, err := GetBlobsPath("")
	if err != nil {
		return err
	}
	if err := fixBlobs(blobsDir); err != nil {
		return err
	}

Michael Yang's avatar
bool  
Michael Yang committed
1220
	if !envconfig.NoPrune() {
1221
1222
1223
1224
1225
1226
1227
		if _, err := Manifests(false); err != nil {
			slog.Warn("corrupt manifests detected, skipping prune operation.  Re-pull or delete to clear", "error", err)
		} else {
			// clean up unused layers and manifests
			if err := PruneLayers(); err != nil {
				return err
			}
1228

1229
1230
1231
1232
			manifestsPath, err := GetManifestPath()
			if err != nil {
				return err
			}
1233

1234
1235
1236
			if err := PruneDirectory(manifestsPath); err != nil {
				return err
			}
1237
1238
1239
		}
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1240
	ctx, done := context.WithCancel(context.Background())
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1241
1242
	schedCtx, schedDone := context.WithCancel(ctx)
	sched := InitScheduler(schedCtx)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1243
	s := &Server{addr: ln.Addr(), sched: sched}
1244
1245

	http.Handle("/", s.GenerateRoutes())
1246

1247
	slog.Info(fmt.Sprintf("Listening on %s (version %s)", ln.Addr(), version.Version))
1248
	srvr := &http.Server{
1249
1250
1251
1252
1253
1254
1255
1256
1257
		// Use http.DefaultServeMux so we get net/http/pprof for
		// free.
		//
		// TODO(bmizerany): Decide if we want to make this
		// configurable so it is not exposed by default, or allow
		// users to bind it to a different port. This was a quick
		// and easy way to get pprof, but it may not be the best
		// way.
		Handler: nil,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1258
1259
	}

1260
1261
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
1262
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
1263
1264
	go func() {
		<-signals
1265
		srvr.Close()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1266
		schedDone()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1267
		sched.unloadAllRunners()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1268
		done()
1269
1270
	}()

1271
1272
1273
1274
	// Locate and log what runners are present at startup
	var runnerNames []string
	for v := range runners.GetAvailableServers() {
		runnerNames = append(runnerNames, v)
1275
	}
1276
1277
	slog.Info("Dynamic LLM libraries", "runners", runnerNames)
	slog.Debug("Override detection logic by setting OLLAMA_LLM_LIBRARY")
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1278

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1279
	s.sched.Run(schedCtx)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1280
1281
1282

	// At startup we retrieve GPU information so we can get log messages before loading a model
	// This will log warnings to the log in case we have problems with detected GPUs
1283
	gpus := discover.GetGPUInfo()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1284
	gpus.LogDetails()
1285

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1286
1287
1288
1289
1290
1291
1292
	err = srvr.Serve(ln)
	// If server is closed from the signal handler, wait for the ctx to be done
	// otherwise error out quickly
	if !errors.Is(err, http.ErrServerClosed) {
		return err
	}
	<-ctx.Done()
1293
	return nil
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1294
}
Michael Yang's avatar
Michael Yang committed
1295

1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
Josh's avatar
Josh committed
1306
1307
1308
1309
			status, ok := r["status"].(int)
			if !ok {
				status = http.StatusInternalServerError
			}
1310
			if errorMsg, ok := r["error"].(string); ok {
Josh's avatar
Josh committed
1311
				c.JSON(status, gin.H{"error": errorMsg})
1312
1313
				return
			} else {
Josh's avatar
Josh committed
1314
				c.JSON(status, gin.H{"error": "unexpected error format in progress response"})
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
1325
func streamResponse(c *gin.Context, ch chan any) {
1326
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
1327
1328
1329
1330
1331
1332
1333
1334
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
1335
			slog.Info(fmt.Sprintf("streamResponse: json.Marshal failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1336
1337
1338
			return false
		}

1339
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
1340
1341
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
1342
			slog.Info(fmt.Sprintf("streamResponse: w.Write failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1343
1344
1345
1346
1347
1348
			return false
		}

		return true
	})
}
Bruce MacDonald's avatar
Bruce MacDonald committed
1349

1350
func (s *Server) PsHandler(c *gin.Context) {
1351
	models := []api.ProcessModelResponse{}
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362

	for _, v := range s.sched.loaded {
		model := v.model
		modelDetails := api.ModelDetails{
			Format:            model.Config.ModelFormat,
			Family:            model.Config.ModelFamily,
			Families:          model.Config.ModelFamilies,
			ParameterSize:     model.Config.ModelType,
			QuantizationLevel: model.Config.FileType,
		}

1363
		mr := api.ProcessModelResponse{
1364
1365
1366
1367
1368
1369
1370
1371
			Model:     model.ShortName,
			Name:      model.ShortName,
			Size:      int64(v.estimatedTotal),
			SizeVRAM:  int64(v.estimatedVRAM),
			Digest:    model.Digest,
			Details:   modelDetails,
			ExpiresAt: v.expiresAt,
		}
1372
1373
1374
1375
1376
1377
1378
1379
		// The scheduler waits to set expiresAt, so if a model is loading it's
		// possible that it will be set to the unix epoch. For those cases, just
		// calculate the time w/ the sessionDuration instead.
		var epoch time.Time
		if v.expiresAt == epoch {
			mr.ExpiresAt = time.Now().Add(v.sessionDuration)
		}

1380
1381
1382
		models = append(models, mr)
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1383
1384
1385
1386
1387
	slices.SortStableFunc(models, func(i, j api.ProcessModelResponse) int {
		// longest duration remaining listed first
		return cmp.Compare(j.ExpiresAt.Unix(), i.ExpiresAt.Unix())
	})

1388
	c.JSON(http.StatusOK, api.ProcessResponse{Models: models})
1389
1390
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1391
func (s *Server) ChatHandler(c *gin.Context) {
1392
1393
	checkpointStart := time.Now()

Bruce MacDonald's avatar
Bruce MacDonald committed
1394
	var req api.ChatRequest
Michael Yang's avatar
Michael Yang committed
1395
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1396
1397
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
1398
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1399
1400
1401
1402
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
	// expire the runner
	if len(req.Messages) == 0 && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
		model, err := GetModel(req.Model)
		if err != nil {
			switch {
			case os.IsNotExist(err):
				c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
			case err.Error() == "invalid model name":
				c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			}
			return
		}
		s.sched.expireRunner(model)

		c.JSON(http.StatusOK, api.ChatResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

Michael Yang's avatar
Michael Yang committed
1429
	caps := []Capability{CapabilityCompletion}
1430
	if len(req.Tools) > 0 {
Michael Yang's avatar
tools  
Michael Yang committed
1431
1432
1433
		caps = append(caps, CapabilityTools)
	}

Michael Yang's avatar
Michael Yang committed
1434
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
1435
1436
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support chat", req.Model)})
Bruce MacDonald's avatar
Bruce MacDonald committed
1437
		return
Michael Yang's avatar
Michael Yang committed
1438
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
1439
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
1440
1441
		return
	}
Michael Yang's avatar
Michael Yang committed
1442

1443
1444
	checkpointLoaded := time.Now()

Michael Yang's avatar
Michael Yang committed
1445
1446
	if len(req.Messages) == 0 {
		c.JSON(http.StatusOK, api.ChatResponse{
1447
			Model:      req.Model,
Michael Yang's avatar
Michael Yang committed
1448
1449
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
1450
1451
			Done:       true,
			DoneReason: "load",
Michael Yang's avatar
Michael Yang committed
1452
		})
1453
1454
1455
		return
	}

Michael Yang's avatar
Michael Yang committed
1456
	msgs := append(m.Messages, req.Messages...)
1457
	if req.Messages[0].Role != "system" && m.System != "" {
Michael Yang's avatar
Michael Yang committed
1458
		msgs = append([]api.Message{{Role: "system", Content: m.System}}, msgs...)
1459
1460
	}

Michael Yang's avatar
Michael Yang committed
1461
	prompt, images, err := chatPrompt(c.Request.Context(), m, r.Tokenize, opts, msgs, req.Tools)
Michael Yang's avatar
Michael Yang committed
1462
	if err != nil {
1463
		slog.Error("chat prompt error", "error", err)
Michael Yang's avatar
Michael Yang committed
1464
1465
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
1466
1467
	}

Michael Yang's avatar
Michael Yang committed
1468
	slog.Debug("chat request", "images", len(images), "prompt", prompt)
1469

Bruce MacDonald's avatar
Bruce MacDonald committed
1470
1471
1472
	ch := make(chan any)
	go func() {
		defer close(ch)
1473
		var sb strings.Builder
1474
		var toolCallIndex int = 0
Michael Yang's avatar
Michael Yang committed
1475
		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
Michael Yang's avatar
Michael Yang committed
1476
1477
1478
			Prompt:  prompt,
			Images:  images,
			Format:  req.Format,
Michael Yang's avatar
Michael Yang committed
1479
			Options: opts,
Michael Yang's avatar
Michael Yang committed
1480
		}, func(r llm.CompletionResponse) {
1481
			res := api.ChatResponse{
1482
1483
1484
1485
1486
				Model:      req.Model,
				CreatedAt:  time.Now().UTC(),
				Message:    api.Message{Role: "assistant", Content: r.Content},
				Done:       r.Done,
				DoneReason: r.DoneReason,
Bruce MacDonald's avatar
Bruce MacDonald committed
1487
1488
1489
1490
1491
1492
1493
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
			}
1494
1495
1496
1497
1498
1499

			if r.Done {
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
			}

1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
			// TODO: tool call checking and filtering should be moved outside of this callback once streaming
			// however this was a simple change for now without reworking streaming logic of this (and other)
			// handlers
			if req.Stream != nil && !*req.Stream || len(req.Tools) == 0 {
				ch <- res
				return
			}

			// Streaming tool calls:
			// If tools are recognized, use a flag to track the sending of a tool downstream
			// This ensures that content is cleared from the message on the last chunk sent
			sb.WriteString(r.Content)
			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
				res.Message.ToolCalls = toolCalls
1514
1515
1516
1517
				for i := range toolCalls {
					toolCalls[i].Function.Index = toolCallIndex
					toolCallIndex++
				}
1518
1519
1520
1521
1522
1523
1524
1525
				res.Message.Content = ""
				sb.Reset()
				ch <- res
				return
			}

			if r.Done {
				// Send any remaining content if no tool calls were detected
1526
				if toolCallIndex == 0 {
1527
1528
1529
1530
					res.Message.Content = sb.String()
				}
				ch <- res
			}
Michael Yang's avatar
Michael Yang committed
1531
		}); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1532
1533
1534
1535
1536
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
tools  
Michael Yang committed
1537
		var resp api.ChatResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
1538
		var sb strings.Builder
Michael Yang's avatar
Michael Yang committed
1539
1540
		for rr := range ch {
			switch t := rr.(type) {
1541
			case api.ChatResponse:
Michael Yang's avatar
Michael Yang committed
1542
				sb.WriteString(t.Message.Content)
Michael Yang's avatar
tools  
Michael Yang committed
1543
				resp = t
1544
			case gin.H:
Michael Yang's avatar
Michael Yang committed
1545
1546
1547
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
1548
				}
Michael Yang's avatar
Michael Yang committed
1549
1550
1551

				c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
				return
1552
			default:
Michael Yang's avatar
Michael Yang committed
1553
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
1554
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
1555
1556
			}
		}
1557

Michael Yang's avatar
tools  
Michael Yang committed
1558
		resp.Message.Content = sb.String()
1559
1560
1561
1562
1563
1564

		if len(req.Tools) > 0 {
			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
				resp.Message.ToolCalls = toolCalls
				resp.Message.Content = ""
			}
Michael Yang's avatar
tools  
Michael Yang committed
1565
1566
1567
		}

		c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
1568
1569
1570
1571
1572
		return
	}

	streamResponse(c, ch)
}
1573

Michael Yang's avatar
Michael Yang committed
1574
func handleScheduleError(c *gin.Context, name string, err error) {
Michael Yang's avatar
Michael Yang committed
1575
	switch {
1576
	case errors.Is(err, errCapabilities), errors.Is(err, errRequired):
Michael Yang's avatar
Michael Yang committed
1577
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
1578
	case errors.Is(err, context.Canceled):
1579
		c.JSON(499, gin.H{"error": "request canceled"})
Michael Yang's avatar
Michael Yang committed
1580
	case errors.Is(err, ErrMaxQueue):
1581
		c.JSON(http.StatusServiceUnavailable, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
1582
1583
	case errors.Is(err, os.ErrNotExist):
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found, try pulling it first", name)})
Michael Yang's avatar
Michael Yang committed
1584
1585
	default:
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
1586
1587
	}
}