routes.go 38.2 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
Michael Yang's avatar
Michael Yang committed
4
	"bytes"
Michael Yang's avatar
Michael Yang committed
5
	"cmp"
6
	"context"
7
	"encoding/binary"
Michael Yang's avatar
Michael Yang committed
8
	"encoding/json"
9
	"errors"
10
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
11
	"io"
12
	"log/slog"
13
	"math"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
14
15
	"net"
	"net/http"
16
	"net/netip"
17
	"os"
18
	"os/signal"
Michael Yang's avatar
Michael Yang committed
19
	"path/filepath"
20
	"slices"
Michael Yang's avatar
Michael Yang committed
21
	"strings"
22
	"syscall"
23
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
24

Michael Yang's avatar
Michael Yang committed
25
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
26
	"github.com/gin-gonic/gin"
27
	"golang.org/x/sync/errgroup"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
28

29
	"github.com/ollama/ollama/api"
30
	"github.com/ollama/ollama/build"
31
	"github.com/ollama/ollama/discover"
32
	"github.com/ollama/ollama/envconfig"
33
34
	"github.com/ollama/ollama/llm"
	"github.com/ollama/ollama/openai"
35
	"github.com/ollama/ollama/parser"
36
	"github.com/ollama/ollama/runners"
37
	"github.com/ollama/ollama/server/imageproc"
Michael Yang's avatar
Michael Yang committed
38
	"github.com/ollama/ollama/template"
39
	"github.com/ollama/ollama/types/errtypes"
Michael Yang's avatar
Michael Yang committed
40
	"github.com/ollama/ollama/types/model"
41
	"github.com/ollama/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
42
43
)

Michael Yang's avatar
Michael Yang committed
44
45
var mode string = gin.DebugMode

46
type Server struct {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
47
48
	addr  net.Addr
	sched *Scheduler
49
50
}

Michael Yang's avatar
Michael Yang committed
51
52
53
54
55
56
57
58
59
60
61
62
func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Michael Yang's avatar
lint  
Michael Yang committed
63
64
65
66
var (
	errRequired    = errors.New("is required")
	errBadTemplate = errors.New("template error")
)
Michael Yang's avatar
Michael Yang committed
67

68
69
70
71
72
73
74
75
76
77
78
func modelOptions(model *Model, requestOpts map[string]interface{}) (api.Options, error) {
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		return api.Options{}, err
	}

	if err := opts.FromMap(requestOpts); err != nil {
		return api.Options{}, err
	}

	return opts, nil
Bruce MacDonald's avatar
Bruce MacDonald committed
79
80
}

Michael Yang's avatar
Michael Yang committed
81
82
83
// scheduleRunner schedules a runner after validating inputs such as capabilities and model options.
// It returns the allocated runner, model instance, and consolidated options if successful and error otherwise.
func (s *Server) scheduleRunner(ctx context.Context, name string, caps []Capability, requestOpts map[string]any, keepAlive *api.Duration) (llm.LlamaServer, *Model, *api.Options, error) {
Michael Yang's avatar
Michael Yang committed
84
	if name == "" {
Michael Yang's avatar
Michael Yang committed
85
		return nil, nil, nil, fmt.Errorf("model %w", errRequired)
Bruce MacDonald's avatar
Bruce MacDonald committed
86
87
	}

Michael Yang's avatar
Michael Yang committed
88
	model, err := GetModel(name)
Bruce MacDonald's avatar
Bruce MacDonald committed
89
	if err != nil {
Michael Yang's avatar
Michael Yang committed
90
		return nil, nil, nil, err
91
92
	}

Michael Yang's avatar
Michael Yang committed
93
	if err := model.CheckCapabilities(caps...); err != nil {
Michael Yang's avatar
Michael Yang committed
94
		return nil, nil, nil, fmt.Errorf("%s %w", name, err)
95
96
	}

Michael Yang's avatar
Michael Yang committed
97
	opts, err := modelOptions(model, requestOpts)
98
	if err != nil {
Michael Yang's avatar
Michael Yang committed
99
		return nil, nil, nil, err
100
101
	}

Michael Yang's avatar
Michael Yang committed
102
	runnerCh, errCh := s.sched.GetRunner(ctx, model, opts, keepAlive)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
103
104
	var runner *runnerRef
	select {
Michael Yang's avatar
Michael Yang committed
105
106
	case runner = <-runnerCh:
	case err = <-errCh:
Michael Yang's avatar
Michael Yang committed
107
		return nil, nil, nil, err
Bruce MacDonald's avatar
Bruce MacDonald committed
108
109
	}

Michael Yang's avatar
Michael Yang committed
110
	return runner.llama, model, &opts, nil
Michael Yang's avatar
Michael Yang committed
111
112
113
}

func (s *Server) GenerateHandler(c *gin.Context) {
114
	checkpointStart := time.Now()
Michael Yang's avatar
Michael Yang committed
115
116
117
118
119
120
	var req api.GenerateRequest
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
121
122
123
		return
	}

124
125
126
127
128
129
130
131
132
133
134
135
136
	model, err := GetModel(req.Model)
	if err != nil {
		switch {
		case os.IsNotExist(err):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		case err.Error() == "invalid model name":
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

Patrick Devine's avatar
Patrick Devine committed
137
138
139
140
141
142
143
144
145
146
147
148
149
150
	// expire the runner
	if req.Prompt == "" && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
		s.sched.expireRunner(model)

		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Response:   "",
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

Michael Yang's avatar
Michael Yang committed
151
152
153
154
155
	if req.Format != "" && req.Format != "json" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be empty or \"json\""})
		return
	} else if req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
Michael Yang's avatar
Michael Yang committed
156
157
158
		return
	}

Michael Yang's avatar
Michael Yang committed
159
	caps := []Capability{CapabilityCompletion}
160
161
162
163
	if req.Suffix != "" {
		caps = append(caps, CapabilityInsert)
	}

Michael Yang's avatar
Michael Yang committed
164
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
165
166
167
168
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support generate", req.Model)})
		return
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
169
170
171
172
		handleScheduleError(c, req.Model, err)
		return
	}

173
174
	checkpointLoaded := time.Now()

175
	// load the model
Michael Yang's avatar
Michael Yang committed
176
177
178
179
180
181
182
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Done:       true,
			DoneReason: "load",
		})
Michael Yang's avatar
Michael Yang committed
183
184
		return
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
185

186
187
188
189
190
191
	isMllama := checkMllamaModelFamily(model)
	if isMllama && len(req.Images) > 1 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "this model only supports one image: more than one image sent"})
		return
	}

Michael Yang's avatar
Michael Yang committed
192
193
	images := make([]llm.ImageData, len(req.Images))
	for i := range req.Images {
194
195
196
197
198
199
200
201
202
203
204
205
206
207
		if isMllama {
			data, aspectRatioID, err := imageproc.Preprocess(req.Images[i])
			if err != nil {
				c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "error processing image"})
				return
			}

			buf := new(bytes.Buffer)
			err = binary.Write(buf, binary.LittleEndian, data)
			if err != nil {
				c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": "error processing image"})
				return
			}

208
			images[i] = llm.ImageData{ID: i, Data: buf.Bytes(), AspectRatioID: aspectRatioID}
209
210
211
		} else {
			images[i] = llm.ImageData{ID: i, Data: req.Images[i]}
		}
Michael Yang's avatar
Michael Yang committed
212
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
213

Michael Yang's avatar
Michael Yang committed
214
215
	prompt := req.Prompt
	if !req.Raw {
Michael Yang's avatar
Michael Yang committed
216
		tmpl := m.Template
Michael Yang's avatar
Michael Yang committed
217
218
219
220
221
222
223
224
		if req.Template != "" {
			tmpl, err = template.Parse(req.Template)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

225
226
227
228
229
230
231
232
233
234
235
236
		var values template.Values
		if req.Suffix != "" {
			values.Prompt = prompt
			values.Suffix = req.Suffix
		} else {
			var msgs []api.Message
			if req.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: req.System})
			} else if m.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: m.System})
			}

Michael Yang's avatar
Michael Yang committed
237
238
239
240
			if req.Context == nil {
				msgs = append(msgs, m.Messages...)
			}

241
			for _, i := range images {
242
				imgPrompt := ""
243
				if isMllama {
244
					imgPrompt = "<|image|>"
245
				}
246
				msgs = append(msgs, api.Message{Role: "user", Content: fmt.Sprintf("[img-%d]"+imgPrompt, i.ID)})
247
248
249
250
251
			}

			values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt})
		}

Michael Yang's avatar
Michael Yang committed
252
253
		var b bytes.Buffer
		if req.Context != nil {
254
			s, err := r.Detokenize(c.Request.Context(), req.Context)
Michael Yang's avatar
Michael Yang committed
255
256
257
258
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
259
			b.WriteString(s)
Michael Yang's avatar
Michael Yang committed
260
		}
261
262
263
264
265
266
267

		if err := tmpl.Execute(&b, values); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		prompt = b.String()
Bruce MacDonald's avatar
Bruce MacDonald committed
268
269
	}

Michael Yang's avatar
Michael Yang committed
270
	slog.Debug("generate request", "prompt", prompt, "images", images)
271

Bruce MacDonald's avatar
Bruce MacDonald committed
272
273
	ch := make(chan any)
	go func() {
274
275
		// TODO (jmorganca): avoid building the response twice both here and below
		var sb strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
276
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
277
		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
Michael Yang's avatar
Michael Yang committed
278
279
280
			Prompt:  prompt,
			Images:  images,
			Format:  req.Format,
Michael Yang's avatar
Michael Yang committed
281
			Options: opts,
282
283
		}, func(cr llm.CompletionResponse) {
			res := api.GenerateResponse{
284
285
				Model:      req.Model,
				CreatedAt:  time.Now().UTC(),
286
287
288
				Response:   cr.Content,
				Done:       cr.Done,
				DoneReason: cr.DoneReason,
Bruce MacDonald's avatar
Bruce MacDonald committed
289
				Metrics: api.Metrics{
290
291
292
293
					PromptEvalCount:    cr.PromptEvalCount,
					PromptEvalDuration: cr.PromptEvalDuration,
					EvalCount:          cr.EvalCount,
					EvalDuration:       cr.EvalDuration,
Bruce MacDonald's avatar
Bruce MacDonald committed
294
				},
Bruce MacDonald's avatar
Bruce MacDonald committed
295
			}
296
297
298
299
300
301
302
303
304
305

			if _, err := sb.WriteString(cr.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
			}

			if cr.Done {
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)

				if !req.Raw {
306
					tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String())
307
308
309
310
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
311
					res.Context = tokens
312
313
314
315
				}
			}

			ch <- res
Michael Yang's avatar
Michael Yang committed
316
		}); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
317
318
319
320
321
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
Michael Yang committed
322
		var r api.GenerateResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
323
		var sb strings.Builder
Michael Yang's avatar
Michael Yang committed
324
325
		for rr := range ch {
			switch t := rr.(type) {
326
			case api.GenerateResponse:
Michael Yang's avatar
Michael Yang committed
327
328
				sb.WriteString(t.Response)
				r = t
329
			case gin.H:
Michael Yang's avatar
Michael Yang committed
330
331
332
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
333
				}
Michael Yang's avatar
Michael Yang committed
334
335
336

				c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
				return
337
			default:
Michael Yang's avatar
Michael Yang committed
338
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
Bruce MacDonald's avatar
Bruce MacDonald committed
339
340
341
				return
			}
		}
342

Michael Yang's avatar
Michael Yang committed
343
344
		r.Response = sb.String()
		c.JSON(http.StatusOK, r)
Bruce MacDonald's avatar
Bruce MacDonald committed
345
346
347
348
349
350
		return
	}

	streamResponse(c, ch)
}

351
func (s *Server) EmbedHandler(c *gin.Context) {
352
	checkpointStart := time.Now()
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
	var req api.EmbedRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	truncate := true

	if req.Truncate != nil && !*req.Truncate {
		truncate = false
	}

	var input []string

	switch i := req.Input.(type) {
	case string:
		if len(i) > 0 {
			input = append(input, i)
		}
	case []any:
		for _, v := range i {
			if _, ok := v.(string); !ok {
				c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
				return
			}
			input = append(input, v.(string))
		}
	default:
386
387
388
389
		if req.Input != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
			return
		}
390
391
392
393
394
395
396
397
	}

	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}

398
399
	checkpointLoaded := time.Now()

400
401
402
403
404
	if len(input) == 0 {
		c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
		return
	}

405
406
407
408
409
410
	kvData, err := getKVData(m.ModelPath, false)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

411
	var count int
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
	for i, s := range input {
		tokens, err := r.Tokenize(c.Request.Context(), s)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		ctxLen := min(opts.NumCtx, int(kvData.ContextLength()))
		if len(tokens) > ctxLen {
			if !truncate {
				c.JSON(http.StatusBadRequest, gin.H{"error": "input length exceeds maximum context length"})
				return
			}

			tokens = tokens[:ctxLen]
			s, err = r.Detokenize(c.Request.Context(), tokens)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

434
435
		count += len(tokens)

436
437
		input[i] = s
	}
438
439
440
441
442
443
444
445
446
447
448
449

	var g errgroup.Group
	embeddings := make([][]float32, len(input))
	for i, text := range input {
		g.Go(func() error {
			embedding, err := r.Embedding(c.Request.Context(), text)
			if err != nil {
				return err
			}
			embeddings[i] = normalize(embedding)
			return nil
		})
450
451
	}

452
453
454
455
	if err := g.Wait(); err != nil {
		slog.Error("embedding generation failed", "error", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embeddings: %v", err)})
		return
456
457
458
	}

	resp := api.EmbedResponse{
459
		Model:           req.Model,
460
		Embeddings:      embeddings,
461
462
		TotalDuration:   time.Since(checkpointStart),
		LoadDuration:    checkpointLoaded.Sub(checkpointStart),
463
		PromptEvalCount: count,
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
	}
	c.JSON(http.StatusOK, resp)
}

func normalize(vec []float32) []float32 {
	var sum float32
	for _, v := range vec {
		sum += v * v
	}

	norm := float32(0.0)
	if sum > 0 {
		norm = float32(1.0 / math.Sqrt(float64(sum)))
	}

	for i := range vec {
		vec[i] *= norm
	}
	return vec
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
485
func (s *Server) EmbeddingsHandler(c *gin.Context) {
Bruce MacDonald's avatar
Bruce MacDonald committed
486
	var req api.EmbeddingRequest
Michael Yang's avatar
Michael Yang committed
487
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
488
489
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
490
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
491
492
493
494
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
495
	r, _, _, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
Bruce MacDonald's avatar
Bruce MacDonald committed
496
	if err != nil {
Michael Yang's avatar
Michael Yang committed
497
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
498
499
500
		return
	}

501
502
503
	// an empty request loads the model
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float64{}})
Bruce MacDonald's avatar
Bruce MacDonald committed
504
505
506
		return
	}

507
	embedding, err := r.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
508
	if err != nil {
509
		slog.Info(fmt.Sprintf("embedding generation failed: %v", err))
Bruce MacDonald's avatar
Bruce MacDonald committed
510
511
512
513
		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
		return
	}

514
515
516
	var e []float64
	for _, v := range embedding {
		e = append(e, float64(v))
517
518
519
	}

	resp := api.EmbeddingResponse{
520
		Embedding: e,
521
522
	}
	c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
523
524
}

525
func (s *Server) PullHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
526
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
527
528
529
530
531
532
533
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
534
535
536
		return
	}

537
538
539
540
541
542
543
544
	name := model.ParseName(cmp.Or(req.Model, req.Name))
	if !name.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid model name"})
		return
	}

	if err := checkNameExists(name); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
545
546
547
		return
	}

548
549
550
	ch := make(chan any)
	go func() {
		defer close(ch)
551
552
		fn := func(r api.ProgressResponse) {
			ch <- r
553
		}
554

Michael Yang's avatar
Michael Yang committed
555
		regOpts := &registryOptions{
556
557
558
			Insecure: req.Insecure,
		}

559
560
561
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

562
		if err := PullModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
563
			ch <- gin.H{"error": err.Error()}
564
565
566
		}
	}()

567
568
569
570
571
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

572
573
574
	streamResponse(c, ch)
}

575
func (s *Server) PushHandler(c *gin.Context) {
576
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
577
578
579
580
581
582
583
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
584
585
		return
	}
Michael Yang's avatar
Michael Yang committed
586

Michael Yang's avatar
Michael Yang committed
587
588
589
590
591
592
593
	var model string
	if req.Model != "" {
		model = req.Model
	} else if req.Name != "" {
		model = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
594
595
596
		return
	}

597
598
599
	ch := make(chan any)
	go func() {
		defer close(ch)
600
601
		fn := func(r api.ProgressResponse) {
			ch <- r
602
		}
603

Michael Yang's avatar
Michael Yang committed
604
		regOpts := &registryOptions{
605
606
607
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
608
609
610
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

Michael Yang's avatar
Michael Yang committed
611
		if err := PushModel(ctx, model, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
612
			ch <- gin.H{"error": err.Error()}
613
614
615
		}
	}()

616
617
618
619
620
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

621
622
623
	streamResponse(c, ch)
}

624
625
626
627
628
629
630
631
func checkNameExists(name model.Name) error {
	names, err := Manifests()
	if err != nil {
		return err
	}

	for n := range names {
		if strings.EqualFold(n.Filepath(), name.Filepath()) && n != name {
Michael Yang's avatar
lint  
Michael Yang committed
632
			return errors.New("a model with that name already exists")
633
634
635
636
637
638
		}
	}

	return nil
}

639
func (s *Server) CreateHandler(c *gin.Context) {
640
641
	var r api.CreateRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
642
643
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
644
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
645
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
646
		return
647
648
	}

649
	name := model.ParseName(cmp.Or(r.Model, r.Name))
Michael Yang's avatar
Michael Yang committed
650
	if !name.IsValid() {
651
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": errtypes.InvalidModelNameErrMsg})
652
653
654
		return
	}

655
656
657
658
659
	if err := checkNameExists(name); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

660
	if r.Path == "" && r.Modelfile == "" {
Michael Yang's avatar
Michael Yang committed
661
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"})
Michael Yang's avatar
Michael Yang committed
662
663
		return
	}
Michael Yang's avatar
Michael Yang committed
664

665
666
667
	var sr io.Reader = strings.NewReader(r.Modelfile)
	if r.Path != "" && r.Modelfile == "" {
		f, err := os.Open(r.Path)
Michael Yang's avatar
Michael Yang committed
668
669
670
671
		if err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)})
			return
		}
Michael Yang's avatar
Michael Yang committed
672
		defer f.Close()
Michael Yang's avatar
Michael Yang committed
673

674
		sr = f
Michael Yang's avatar
Michael Yang committed
675
	}
Michael Yang's avatar
Michael Yang committed
676

677
	f, err := parser.ParseFile(sr)
Michael Yang's avatar
Michael Yang committed
678
679
680
681
682
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
683
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
684
685
	go func() {
		defer close(ch)
686
687
		fn := func(resp api.ProgressResponse) {
			ch <- resp
688
689
		}

690
691
692
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

693
		quantization := cmp.Or(r.Quantize, r.Quantization)
Josh's avatar
Josh committed
694
695
696
		if err := CreateModel(ctx, name, filepath.Dir(r.Path), strings.ToUpper(quantization), f, fn); errors.Is(err, errBadTemplate) {
			ch <- gin.H{"error": err.Error(), "status": http.StatusBadRequest}
		} else if err != nil {
Michael Yang's avatar
Michael Yang committed
697
			ch <- gin.H{"error": err.Error()}
698
		}
Michael Yang's avatar
Michael Yang committed
699
	}()
Michael Yang's avatar
Michael Yang committed
700

701
	if r.Stream != nil && !*r.Stream {
702
703
704
705
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
706
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
707
708
}

709
func (s *Server) DeleteHandler(c *gin.Context) {
710
711
	var r api.DeleteRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
712
713
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
714
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
715
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
716
717
718
		return
	}

719
720
721
	n := model.ParseName(cmp.Or(r.Model, r.Name))
	if !n.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("name %q is invalid", cmp.Or(r.Model, r.Name))})
722
723
		return
	}
Michael Yang's avatar
Michael Yang committed
724

725
	m, err := ParseNamedManifest(n)
Michael Yang's avatar
Michael Yang committed
726
	if err != nil {
727
728
729
730
731
732
		switch {
		case os.IsNotExist(err):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", cmp.Or(r.Model, r.Name))})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
Michael Yang's avatar
Michael Yang committed
733
734
735
		return
	}

736
	if err := m.Remove(); err != nil {
Michael Yang's avatar
Michael Yang committed
737
738
739
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
740
741
742
743
744

	if err := m.RemoveLayers(); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
745
746
}

747
func (s *Server) ShowHandler(c *gin.Context) {
Patrick Devine's avatar
Patrick Devine committed
748
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
749
750
751
752
753
754
755
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
756
757
758
		return
	}

Michael Yang's avatar
Michael Yang committed
759
	if req.Model != "" {
Michael Yang's avatar
Michael Yang committed
760
		// noop
Michael Yang's avatar
Michael Yang committed
761
	} else if req.Name != "" {
Michael Yang's avatar
Michael Yang committed
762
		req.Model = req.Name
Michael Yang's avatar
Michael Yang committed
763
	} else {
764
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
765
766
767
		return
	}

768
	resp, err := GetModelInfo(req)
Patrick Devine's avatar
Patrick Devine committed
769
	if err != nil {
770
771
		switch {
		case os.IsNotExist(err):
Michael Yang's avatar
Michael Yang committed
772
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
773
774
775
		case err.Error() == "invalid model name":
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
Patrick Devine's avatar
Patrick Devine committed
776
777
778
779
780
781
782
783
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

784
func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
785
	m, err := GetModel(req.Model)
Patrick Devine's avatar
Patrick Devine committed
786
787
788
789
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
790
	modelDetails := api.ModelDetails{
791
792
793
794
795
796
		ParentModel:       m.ParentModel,
		Format:            m.Config.ModelFormat,
		Family:            m.Config.ModelFamily,
		Families:          m.Config.ModelFamilies,
		ParameterSize:     m.Config.ModelType,
		QuantizationLevel: m.Config.FileType,
Patrick Devine's avatar
Patrick Devine committed
797
798
	}

799
	if req.System != "" {
800
		m.System = req.System
801
802
	}

Michael Yang's avatar
Michael Yang committed
803
804
805
	msgs := make([]api.Message, len(m.Messages))
	for i, msg := range m.Messages {
		msgs[i] = api.Message{Role: msg.Role, Content: msg.Content}
806
807
	}

808
809
	n := model.ParseName(req.Model)
	if !n.IsValid() {
Michael Yang's avatar
lint  
Michael Yang committed
810
		return nil, errors.New("invalid model name")
811
812
813
814
815
816
817
	}

	manifest, err := ParseNamedManifest(n)
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
818
	resp := &api.ShowResponse{
819
820
		License:    strings.Join(m.License, "\n"),
		System:     m.System,
Michael Yang's avatar
Michael Yang committed
821
		Template:   m.Template.String(),
822
823
824
		Details:    modelDetails,
		Messages:   msgs,
		ModifiedAt: manifest.fi.ModTime(),
Patrick Devine's avatar
Patrick Devine committed
825
826
827
828
	}

	var params []string
	cs := 30
829
	for k, v := range m.Options {
Patrick Devine's avatar
Patrick Devine committed
830
831
832
		switch val := v.(type) {
		case []interface{}:
			for _, nv := range val {
Patrick Devine's avatar
Patrick Devine committed
833
				params = append(params, fmt.Sprintf("%-*s %#v", cs, k, nv))
Patrick Devine's avatar
Patrick Devine committed
834
			}
Patrick Devine's avatar
Patrick Devine committed
835
836
		default:
			params = append(params, fmt.Sprintf("%-*s %#v", cs, k, v))
Patrick Devine's avatar
Patrick Devine committed
837
838
839
840
		}
	}
	resp.Parameters = strings.Join(params, "\n")

841
842
	for k, v := range req.Options {
		if _, ok := req.Options[k]; ok {
843
			m.Options[k] = v
844
845
846
		}
	}

847
	var sb strings.Builder
848
	fmt.Fprintln(&sb, "# Modelfile generated by \"ollama show\"")
849
	fmt.Fprintln(&sb, "# To build a new Modelfile based on this, replace FROM with:")
850
851
	fmt.Fprintf(&sb, "# FROM %s\n\n", m.ShortName)
	fmt.Fprint(&sb, m.String())
852
	resp.Modelfile = sb.String()
853

854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
	kvData, err := getKVData(m.ModelPath, req.Verbose)
	if err != nil {
		return nil, err
	}
	delete(kvData, "general.name")
	delete(kvData, "tokenizer.chat_template")
	resp.ModelInfo = kvData

	if len(m.ProjectorPaths) > 0 {
		projectorData, err := getKVData(m.ProjectorPaths[0], req.Verbose)
		if err != nil {
			return nil, err
		}
		resp.ProjectorInfo = projectorData
	}

Patrick Devine's avatar
Patrick Devine committed
870
871
872
	return resp, nil
}

873
func getKVData(digest string, verbose bool) (llm.KV, error) {
874
875
876
877
878
	maxArraySize := 0
	if verbose {
		maxArraySize = -1
	}
	kvData, err := llm.LoadModel(digest, maxArraySize)
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
	if err != nil {
		return nil, err
	}

	kv := kvData.KV()

	if !verbose {
		for k := range kv {
			if t, ok := kv[k].([]any); len(t) > 5 && ok {
				kv[k] = []any{}
			}
		}
	}

	return kv, nil
}

896
func (s *Server) ListHandler(c *gin.Context) {
897
	ms, err := Manifests()
Patrick Devine's avatar
Patrick Devine committed
898
899
900
901
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
902

903
	models := []api.ListModelResponse{}
904
905
	for n, m := range ms {
		var cf ConfigV2
906
907
908
909
910
911
912
913
914
915
916
917
918

		if m.Config.Digest != "" {
			f, err := m.Config.Open()
			if err != nil {
				slog.Warn("bad manifest filepath", "name", n, "error", err)
				continue
			}
			defer f.Close()

			if err := json.NewDecoder(f).Decode(&cf); err != nil {
				slog.Warn("bad manifest config", "name", n, "error", err)
				continue
			}
Patrick Devine's avatar
Patrick Devine committed
919
		}
Michael Yang's avatar
Michael Yang committed
920

921
		// tag should never be masked
922
		models = append(models, api.ListModelResponse{
923
924
925
926
927
928
929
930
931
932
933
934
935
			Model:      n.DisplayShortest(),
			Name:       n.DisplayShortest(),
			Size:       m.Size(),
			Digest:     m.digest,
			ModifiedAt: m.fi.ModTime(),
			Details: api.ModelDetails{
				Format:            cf.ModelFormat,
				Family:            cf.ModelFamily,
				Families:          cf.ModelFamilies,
				ParameterSize:     cf.ModelType,
				QuantizationLevel: cf.FileType,
			},
		})
Patrick Devine's avatar
Patrick Devine committed
936
937
	}

938
	slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
939
940
941
942
		// most recently modified first
		return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
	})

Michael Yang's avatar
Michael Yang committed
943
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
944
945
}

946
func (s *Server) CopyHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
947
948
	var r api.CopyRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
949
950
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
951
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
952
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
953
954
955
		return
	}

Michael Yang's avatar
Michael Yang committed
956
957
	src := model.ParseName(r.Source)
	if !src.IsValid() {
Michael Yang's avatar
Michael Yang committed
958
959
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("source %q is invalid", r.Source)})
		return
960
961
	}

Michael Yang's avatar
Michael Yang committed
962
963
	dst := model.ParseName(r.Destination)
	if !dst.IsValid() {
964
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("destination %q is invalid", r.Destination)})
Patrick Devine's avatar
Patrick Devine committed
965
966
		return
	}
Michael Yang's avatar
Michael Yang committed
967

968
969
970
971
972
	if err := checkNameExists(dst); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
973
974
975
976
977
	if err := CopyModel(src, dst); errors.Is(err, os.ErrNotExist) {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
	} else if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
	}
Patrick Devine's avatar
Patrick Devine committed
978
979
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
980
func (s *Server) HeadBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
981
982
983
984
985
986
987
988
989
990
991
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); err != nil {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	}

Michael Yang's avatar
Michael Yang committed
992
	c.Status(http.StatusOK)
Michael Yang's avatar
Michael Yang committed
993
994
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
995
func (s *Server) CreateBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
996
997
	if ib, ok := intermediateBlobs[c.Param("digest")]; ok {
		p, err := GetBlobsPath(ib)
998
999
1000
1001
1002
1003
		if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
Michael Yang's avatar
Michael Yang committed
1004
1005
			slog.Info("evicting intermediate blob which no longer exists", "digest", ib)
			delete(intermediateBlobs, c.Param("digest"))
1006
1007
1008
1009
1010
1011
1012
1013
1014
		} else if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		} else {
			c.Status(http.StatusOK)
			return
		}
	}

1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	_, err = os.Stat(path)
	switch {
	case errors.Is(err, os.ErrNotExist):
		// noop
	case err != nil:
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	default:
		c.Status(http.StatusOK)
		return
	}

1033
	layer, err := NewLayer(c.Request.Body, "")
Michael Yang's avatar
Michael Yang committed
1034
1035
1036
1037
1038
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

1039
1040
	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
Michael Yang's avatar
Michael Yang committed
1041
1042
1043
		return
	}

Michael Yang's avatar
Michael Yang committed
1044
	c.Status(http.StatusCreated)
Michael Yang's avatar
Michael Yang committed
1045
1046
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
func isLocalIP(ip netip.Addr) bool {
	if interfaces, err := net.Interfaces(); err == nil {
		for _, iface := range interfaces {
			addrs, err := iface.Addrs()
			if err != nil {
				continue
			}

			for _, a := range addrs {
				if parsed, _, err := net.ParseCIDR(a.String()); err == nil {
					if parsed.String() == ip.String() {
						return true
					}
				}
			}
		}
	}

	return false
}

1068
func allowedHost(host string) bool {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1069
	if host == "" || host == "localhost" {
1070
1071
1072
1073
1074
1075
1076
		return true
	}

	if hostname, err := os.Hostname(); err == nil && host == hostname {
		return true
	}

Michael Yang's avatar
lint  
Michael Yang committed
1077
	tlds := []string{
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1078
1079
1080
		"localhost",
		"local",
		"internal",
1081
	}
1082

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1083
	// check if the host is a local TLD
1084
1085
1086
1087
1088
1089
	for _, tld := range tlds {
		if strings.HasSuffix(host, "."+tld) {
			return true
		}
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1090
	return false
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1091
}
1092

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1093
1094
1095
func allowedHostsMiddleware(addr net.Addr) gin.HandlerFunc {
	return func(c *gin.Context) {
		if addr == nil {
1096
1097
1098
1099
			c.Next()
			return
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1100
		if addr, err := netip.ParseAddrPort(addr.String()); err == nil && !addr.Addr().IsLoopback() {
1101
1102
1103
1104
1105
1106
1107
1108
1109
			c.Next()
			return
		}

		host, _, err := net.SplitHostPort(c.Request.Host)
		if err != nil {
			host = c.Request.Host
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1110
		if addr, err := netip.ParseAddr(host); err == nil {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1111
			if addr.IsLoopback() || addr.IsPrivate() || addr.IsUnspecified() || isLocalIP(addr) {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1112
1113
1114
1115
1116
				c.Next()
				return
			}
		}

1117
		if allowedHost(host) {
Michael Yang's avatar
lint  
Michael Yang committed
1118
			if c.Request.Method == http.MethodOptions {
1119
1120
1121
1122
				c.AbortWithStatus(http.StatusNoContent)
				return
			}

1123
1124
1125
1126
1127
1128
			c.Next()
			return
		}

		c.AbortWithStatus(http.StatusForbidden)
	}
1129
}
1130

1131
func (s *Server) GenerateRoutes() http.Handler {
Michael Yang's avatar
Michael Yang committed
1132
1133
	config := cors.DefaultConfig()
	config.AllowWildcard = true
1134
	config.AllowBrowserExtensions = true
1135
	config.AllowHeaders = []string{"Authorization", "Content-Type", "User-Agent", "Accept", "X-Requested-With"}
royjhan's avatar
royjhan committed
1136
1137
1138
1139
	openAIProperties := []string{"lang", "package-version", "os", "arch", "runtime", "runtime-version", "async"}
	for _, prop := range openAIProperties {
		config.AllowHeaders = append(config.AllowHeaders, "x-stainless-"+prop)
	}
Michael Yang's avatar
origins  
Michael Yang committed
1140
	config.AllowOrigins = envconfig.Origins()
Michael Yang's avatar
Michael Yang committed
1141

Bruce MacDonald's avatar
Bruce MacDonald committed
1142
	r := gin.Default()
1143
1144
	r.Use(
		cors.New(config),
1145
		allowedHostsMiddleware(s.addr),
1146
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
1147

1148
	r.POST("/api/pull", s.PullHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1149
1150
	r.POST("/api/generate", s.GenerateHandler)
	r.POST("/api/chat", s.ChatHandler)
1151
	r.POST("/api/embed", s.EmbedHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1152
	r.POST("/api/embeddings", s.EmbeddingsHandler)
1153
1154
1155
1156
1157
	r.POST("/api/create", s.CreateHandler)
	r.POST("/api/push", s.PushHandler)
	r.POST("/api/copy", s.CopyHandler)
	r.DELETE("/api/delete", s.DeleteHandler)
	r.POST("/api/show", s.ShowHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1158
1159
	r.POST("/api/blobs/:digest", s.CreateBlobHandler)
	r.HEAD("/api/blobs/:digest", s.HeadBlobHandler)
1160
	r.GET("/api/ps", s.PsHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1161

1162
	// Compatibility endpoints
1163
	r.POST("/v1/chat/completions", openai.ChatMiddleware(), s.ChatHandler)
1164
	r.POST("/v1/completions", openai.CompletionsMiddleware(), s.GenerateHandler)
1165
	r.POST("/v1/embeddings", openai.EmbeddingsMiddleware(), s.EmbedHandler)
1166
1167
	r.GET("/v1/models", openai.ListMiddleware(), s.ListHandler)
	r.GET("/v1/models/:model", openai.RetrieveMiddleware(), s.ShowHandler)
1168

Michael Yang's avatar
Michael Yang committed
1169
1170
1171
1172
1173
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

1174
		r.Handle(method, "/api/tags", s.ListHandler)
Michael Yang's avatar
Michael Yang committed
1175
1176
1177
		r.Handle(method, "/api/version", func(c *gin.Context) {
			c.JSON(http.StatusOK, gin.H{"version": version.Version})
		})
Michael Yang's avatar
Michael Yang committed
1178
1179
	}

1180
1181
1182
1183
	return r
}

func Serve(ln net.Listener) error {
Michael Yang's avatar
Michael Yang committed
1184
	level := slog.LevelInfo
Michael Yang's avatar
Michael Yang committed
1185
	if envconfig.Debug() {
Michael Yang's avatar
Michael Yang committed
1186
		level = slog.LevelDebug
1187
	}
Michael Yang's avatar
Michael Yang committed
1188

1189
	slog.Info("server config", "env", envconfig.Values())
Michael Yang's avatar
Michael Yang committed
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
	handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
		Level:     level,
		AddSource: true,
		ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr {
			if attr.Key == slog.SourceKey {
				source := attr.Value.Any().(*slog.Source)
				source.File = filepath.Base(source.File)
			}

			return attr
		},
	})

	slog.SetDefault(slog.New(handler))

1205
1206
1207
1208
1209
1210
1211
1212
	blobsDir, err := GetBlobsPath("")
	if err != nil {
		return err
	}
	if err := fixBlobs(blobsDir); err != nil {
		return err
	}

Michael Yang's avatar
bool  
Michael Yang committed
1213
	if !envconfig.NoPrune() {
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
		// clean up unused layers and manifests
		if err := PruneLayers(); err != nil {
			return err
		}

		manifestsPath, err := GetManifestPath()
		if err != nil {
			return err
		}

		if err := PruneDirectory(manifestsPath); err != nil {
			return err
		}
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1229
	ctx, done := context.WithCancel(context.Background())
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1230
1231
	schedCtx, schedDone := context.WithCancel(ctx)
	sched := InitScheduler(schedCtx)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1232
	s := &Server{addr: ln.Addr(), sched: sched}
1233
1234

	http.Handle("/", s.GenerateRoutes())
1235

1236
	slog.Info(fmt.Sprintf("Listening on %s (version %s)", ln.Addr(), version.Version))
1237
	srvr := &http.Server{
1238
1239
1240
1241
1242
1243
1244
1245
1246
		// Use http.DefaultServeMux so we get net/http/pprof for
		// free.
		//
		// TODO(bmizerany): Decide if we want to make this
		// configurable so it is not exposed by default, or allow
		// users to bind it to a different port. This was a quick
		// and easy way to get pprof, but it may not be the best
		// way.
		Handler: nil,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1247
1248
	}

1249
1250
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
1251
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
1252
1253
	go func() {
		<-signals
1254
		srvr.Close()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1255
		schedDone()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1256
		sched.unloadAllRunners()
1257
		runners.Cleanup(build.EmbedFS)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1258
		done()
1259
1260
	}()

1261
1262
	if _, err := runners.Refresh(build.EmbedFS); err != nil {
		return fmt.Errorf("unable to initialize llm runners %w", err)
1263
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1264

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1265
	s.sched.Run(schedCtx)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1266
1267
1268

	// At startup we retrieve GPU information so we can get log messages before loading a model
	// This will log warnings to the log in case we have problems with detected GPUs
1269
	gpus := discover.GetGPUInfo()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1270
	gpus.LogDetails()
1271

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1272
1273
1274
1275
1276
1277
1278
	err = srvr.Serve(ln)
	// If server is closed from the signal handler, wait for the ctx to be done
	// otherwise error out quickly
	if !errors.Is(err, http.ErrServerClosed) {
		return err
	}
	<-ctx.Done()
1279
	return nil
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1280
}
Michael Yang's avatar
Michael Yang committed
1281

1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
Josh's avatar
Josh committed
1292
1293
1294
1295
			status, ok := r["status"].(int)
			if !ok {
				status = http.StatusInternalServerError
			}
1296
			if errorMsg, ok := r["error"].(string); ok {
Josh's avatar
Josh committed
1297
				c.JSON(status, gin.H{"error": errorMsg})
1298
1299
				return
			} else {
Josh's avatar
Josh committed
1300
				c.JSON(status, gin.H{"error": "unexpected error format in progress response"})
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
1311
func streamResponse(c *gin.Context, ch chan any) {
1312
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
1313
1314
1315
1316
1317
1318
1319
1320
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
1321
			slog.Info(fmt.Sprintf("streamResponse: json.Marshal failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1322
1323
1324
			return false
		}

1325
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
1326
1327
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
1328
			slog.Info(fmt.Sprintf("streamResponse: w.Write failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1329
1330
1331
1332
1333
1334
			return false
		}

		return true
	})
}
Bruce MacDonald's avatar
Bruce MacDonald committed
1335

1336
func (s *Server) PsHandler(c *gin.Context) {
1337
	models := []api.ProcessModelResponse{}
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348

	for _, v := range s.sched.loaded {
		model := v.model
		modelDetails := api.ModelDetails{
			Format:            model.Config.ModelFormat,
			Family:            model.Config.ModelFamily,
			Families:          model.Config.ModelFamilies,
			ParameterSize:     model.Config.ModelType,
			QuantizationLevel: model.Config.FileType,
		}

1349
		mr := api.ProcessModelResponse{
1350
1351
1352
1353
1354
1355
1356
1357
			Model:     model.ShortName,
			Name:      model.ShortName,
			Size:      int64(v.estimatedTotal),
			SizeVRAM:  int64(v.estimatedVRAM),
			Digest:    model.Digest,
			Details:   modelDetails,
			ExpiresAt: v.expiresAt,
		}
1358
1359
1360
1361
1362
1363
1364
1365
		// The scheduler waits to set expiresAt, so if a model is loading it's
		// possible that it will be set to the unix epoch. For those cases, just
		// calculate the time w/ the sessionDuration instead.
		var epoch time.Time
		if v.expiresAt == epoch {
			mr.ExpiresAt = time.Now().Add(v.sessionDuration)
		}

1366
1367
1368
		models = append(models, mr)
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1369
1370
1371
1372
1373
	slices.SortStableFunc(models, func(i, j api.ProcessModelResponse) int {
		// longest duration remaining listed first
		return cmp.Compare(j.ExpiresAt.Unix(), i.ExpiresAt.Unix())
	})

1374
	c.JSON(http.StatusOK, api.ProcessResponse{Models: models})
1375
1376
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1377
func (s *Server) ChatHandler(c *gin.Context) {
1378
1379
	checkpointStart := time.Now()

Bruce MacDonald's avatar
Bruce MacDonald committed
1380
	var req api.ChatRequest
Michael Yang's avatar
Michael Yang committed
1381
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1382
1383
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
1384
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1385
1386
1387
1388
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
	// expire the runner
	if len(req.Messages) == 0 && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
		model, err := GetModel(req.Model)
		if err != nil {
			switch {
			case os.IsNotExist(err):
				c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
			case err.Error() == "invalid model name":
				c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			}
			return
		}
		s.sched.expireRunner(model)

		c.JSON(http.StatusOK, api.ChatResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

Michael Yang's avatar
Michael Yang committed
1415
	caps := []Capability{CapabilityCompletion}
1416
	if len(req.Tools) > 0 {
Michael Yang's avatar
tools  
Michael Yang committed
1417
1418
1419
		caps = append(caps, CapabilityTools)
	}

Michael Yang's avatar
Michael Yang committed
1420
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
1421
1422
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support chat", req.Model)})
Bruce MacDonald's avatar
Bruce MacDonald committed
1423
		return
Michael Yang's avatar
Michael Yang committed
1424
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
1425
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
1426
1427
		return
	}
Michael Yang's avatar
Michael Yang committed
1428

1429
1430
	checkpointLoaded := time.Now()

Michael Yang's avatar
Michael Yang committed
1431
1432
	if len(req.Messages) == 0 {
		c.JSON(http.StatusOK, api.ChatResponse{
1433
			Model:      req.Model,
Michael Yang's avatar
Michael Yang committed
1434
1435
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
1436
1437
			Done:       true,
			DoneReason: "load",
Michael Yang's avatar
Michael Yang committed
1438
		})
1439
1440
1441
		return
	}

Michael Yang's avatar
Michael Yang committed
1442
	msgs := append(m.Messages, req.Messages...)
1443
	if req.Messages[0].Role != "system" && m.System != "" {
Michael Yang's avatar
Michael Yang committed
1444
		msgs = append([]api.Message{{Role: "system", Content: m.System}}, msgs...)
1445
1446
	}

Michael Yang's avatar
Michael Yang committed
1447
	prompt, images, err := chatPrompt(c.Request.Context(), m, r.Tokenize, opts, msgs, req.Tools)
Michael Yang's avatar
Michael Yang committed
1448
1449
1450
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
1451
1452
	}

Michael Yang's avatar
Michael Yang committed
1453
	slog.Debug("chat request", "images", len(images), "prompt", prompt)
1454

Bruce MacDonald's avatar
Bruce MacDonald committed
1455
1456
1457
	ch := make(chan any)
	go func() {
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
1458
		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
Michael Yang's avatar
Michael Yang committed
1459
1460
1461
			Prompt:  prompt,
			Images:  images,
			Format:  req.Format,
Michael Yang's avatar
Michael Yang committed
1462
			Options: opts,
Michael Yang's avatar
Michael Yang committed
1463
		}, func(r llm.CompletionResponse) {
1464
			res := api.ChatResponse{
1465
1466
1467
1468
1469
				Model:      req.Model,
				CreatedAt:  time.Now().UTC(),
				Message:    api.Message{Role: "assistant", Content: r.Content},
				Done:       r.Done,
				DoneReason: r.DoneReason,
Bruce MacDonald's avatar
Bruce MacDonald committed
1470
1471
1472
1473
1474
1475
1476
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
			}
1477
1478
1479
1480
1481
1482
1483

			if r.Done {
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
			}

			ch <- res
Michael Yang's avatar
Michael Yang committed
1484
		}); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1485
1486
1487
1488
1489
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
tools  
Michael Yang committed
1490
		var resp api.ChatResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
1491
		var sb strings.Builder
Michael Yang's avatar
Michael Yang committed
1492
1493
		for rr := range ch {
			switch t := rr.(type) {
1494
			case api.ChatResponse:
Michael Yang's avatar
Michael Yang committed
1495
				sb.WriteString(t.Message.Content)
Michael Yang's avatar
tools  
Michael Yang committed
1496
				resp = t
1497
			case gin.H:
Michael Yang's avatar
Michael Yang committed
1498
1499
1500
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
1501
				}
Michael Yang's avatar
Michael Yang committed
1502
1503
1504

				c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
				return
1505
			default:
Michael Yang's avatar
Michael Yang committed
1506
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
1507
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
1508
1509
			}
		}
1510

Michael Yang's avatar
tools  
Michael Yang committed
1511
		resp.Message.Content = sb.String()
1512
1513
1514
1515
1516
1517

		if len(req.Tools) > 0 {
			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
				resp.Message.ToolCalls = toolCalls
				resp.Message.Content = ""
			}
Michael Yang's avatar
tools  
Michael Yang committed
1518
1519
1520
		}

		c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
1521
1522
1523
1524
1525
		return
	}

	streamResponse(c, ch)
}
1526

Michael Yang's avatar
Michael Yang committed
1527
func handleScheduleError(c *gin.Context, name string, err error) {
Michael Yang's avatar
Michael Yang committed
1528
	switch {
1529
	case errors.Is(err, errCapabilities), errors.Is(err, errRequired):
Michael Yang's avatar
Michael Yang committed
1530
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
1531
	case errors.Is(err, context.Canceled):
1532
		c.JSON(499, gin.H{"error": "request canceled"})
Michael Yang's avatar
Michael Yang committed
1533
	case errors.Is(err, ErrMaxQueue):
1534
		c.JSON(http.StatusServiceUnavailable, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
1535
1536
	case errors.Is(err, os.ErrNotExist):
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found, try pulling it first", name)})
Michael Yang's avatar
Michael Yang committed
1537
1538
	default:
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
1539
1540
	}
}