"examples/trials/toy/config.yml" did not exist on "eef6bd35e3a634866236df7dde6d9b26d2b00c56"
routes.go 37.1 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
Michael Yang's avatar
Michael Yang committed
4
	"bytes"
Michael Yang's avatar
Michael Yang committed
5
	"cmp"
6
	"context"
Michael Yang's avatar
Michael Yang committed
7
	"encoding/json"
8
	"errors"
9
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
10
	"io"
11
	"log/slog"
12
	"math"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
13
14
	"net"
	"net/http"
15
	"net/netip"
16
	"os"
17
	"os/signal"
Michael Yang's avatar
Michael Yang committed
18
	"path/filepath"
19
	"slices"
Michael Yang's avatar
Michael Yang committed
20
	"strings"
21
	"syscall"
22
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
23

Michael Yang's avatar
Michael Yang committed
24
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
25
	"github.com/gin-gonic/gin"
26
	"golang.org/x/sync/errgroup"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
27

28
	"github.com/ollama/ollama/api"
29
	"github.com/ollama/ollama/build"
30
	"github.com/ollama/ollama/envconfig"
31
32
33
	"github.com/ollama/ollama/gpu"
	"github.com/ollama/ollama/llm"
	"github.com/ollama/ollama/openai"
34
	"github.com/ollama/ollama/parser"
35
	"github.com/ollama/ollama/runners"
Michael Yang's avatar
Michael Yang committed
36
	"github.com/ollama/ollama/template"
37
	"github.com/ollama/ollama/types/errtypes"
Michael Yang's avatar
Michael Yang committed
38
	"github.com/ollama/ollama/types/model"
39
	"github.com/ollama/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
40
41
)

Michael Yang's avatar
Michael Yang committed
42
43
var mode string = gin.DebugMode

44
type Server struct {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
45
46
	addr  net.Addr
	sched *Scheduler
47
48
}

Michael Yang's avatar
Michael Yang committed
49
50
51
52
53
54
55
56
57
58
59
60
func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Michael Yang's avatar
lint  
Michael Yang committed
61
62
63
64
var (
	errRequired    = errors.New("is required")
	errBadTemplate = errors.New("template error")
)
Michael Yang's avatar
Michael Yang committed
65

66
67
68
69
70
71
72
73
74
75
76
func modelOptions(model *Model, requestOpts map[string]interface{}) (api.Options, error) {
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		return api.Options{}, err
	}

	if err := opts.FromMap(requestOpts); err != nil {
		return api.Options{}, err
	}

	return opts, nil
Bruce MacDonald's avatar
Bruce MacDonald committed
77
78
}

Michael Yang's avatar
Michael Yang committed
79
80
81
// scheduleRunner schedules a runner after validating inputs such as capabilities and model options.
// It returns the allocated runner, model instance, and consolidated options if successful and error otherwise.
func (s *Server) scheduleRunner(ctx context.Context, name string, caps []Capability, requestOpts map[string]any, keepAlive *api.Duration) (llm.LlamaServer, *Model, *api.Options, error) {
Michael Yang's avatar
Michael Yang committed
82
	if name == "" {
Michael Yang's avatar
Michael Yang committed
83
		return nil, nil, nil, fmt.Errorf("model %w", errRequired)
Bruce MacDonald's avatar
Bruce MacDonald committed
84
85
	}

Michael Yang's avatar
Michael Yang committed
86
	model, err := GetModel(name)
Bruce MacDonald's avatar
Bruce MacDonald committed
87
	if err != nil {
Michael Yang's avatar
Michael Yang committed
88
		return nil, nil, nil, err
89
90
	}

Michael Yang's avatar
Michael Yang committed
91
	if err := model.CheckCapabilities(caps...); err != nil {
Michael Yang's avatar
Michael Yang committed
92
		return nil, nil, nil, fmt.Errorf("%s %w", name, err)
93
94
	}

Michael Yang's avatar
Michael Yang committed
95
	opts, err := modelOptions(model, requestOpts)
96
	if err != nil {
Michael Yang's avatar
Michael Yang committed
97
		return nil, nil, nil, err
98
99
	}

Michael Yang's avatar
Michael Yang committed
100
	runnerCh, errCh := s.sched.GetRunner(ctx, model, opts, keepAlive)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
101
102
	var runner *runnerRef
	select {
Michael Yang's avatar
Michael Yang committed
103
104
	case runner = <-runnerCh:
	case err = <-errCh:
Michael Yang's avatar
Michael Yang committed
105
		return nil, nil, nil, err
Bruce MacDonald's avatar
Bruce MacDonald committed
106
107
	}

Michael Yang's avatar
Michael Yang committed
108
	return runner.llama, model, &opts, nil
Michael Yang's avatar
Michael Yang committed
109
110
111
}

func (s *Server) GenerateHandler(c *gin.Context) {
112
	checkpointStart := time.Now()
Michael Yang's avatar
Michael Yang committed
113
114
115
116
117
118
	var req api.GenerateRequest
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
119
120
121
		return
	}

Patrick Devine's avatar
Patrick Devine committed
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
	// expire the runner
	if req.Prompt == "" && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
		model, err := GetModel(req.Model)
		if err != nil {
			switch {
			case os.IsNotExist(err):
				c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
			case err.Error() == "invalid model name":
				c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			}
			return
		}
		s.sched.expireRunner(model)

		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Response:   "",
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

Michael Yang's avatar
Michael Yang committed
148
149
150
151
152
	if req.Format != "" && req.Format != "json" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be empty or \"json\""})
		return
	} else if req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
Michael Yang's avatar
Michael Yang committed
153
154
155
		return
	}

Michael Yang's avatar
Michael Yang committed
156
	caps := []Capability{CapabilityCompletion}
157
158
159
160
	if req.Suffix != "" {
		caps = append(caps, CapabilityInsert)
	}

Michael Yang's avatar
Michael Yang committed
161
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
162
163
164
165
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support generate", req.Model)})
		return
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
166
167
168
169
		handleScheduleError(c, req.Model, err)
		return
	}

170
171
	checkpointLoaded := time.Now()

Michael Yang's avatar
Michael Yang committed
172
173
174
175
176
177
178
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Done:       true,
			DoneReason: "load",
		})
Michael Yang's avatar
Michael Yang committed
179
180
		return
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
181

Michael Yang's avatar
Michael Yang committed
182
183
184
185
	images := make([]llm.ImageData, len(req.Images))
	for i := range req.Images {
		images[i] = llm.ImageData{ID: i, Data: req.Images[i]}
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
186

Michael Yang's avatar
Michael Yang committed
187
188
	prompt := req.Prompt
	if !req.Raw {
Michael Yang's avatar
Michael Yang committed
189
		tmpl := m.Template
Michael Yang's avatar
Michael Yang committed
190
191
192
193
194
195
196
197
		if req.Template != "" {
			tmpl, err = template.Parse(req.Template)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

198
199
200
201
202
203
204
205
206
207
208
209
		var values template.Values
		if req.Suffix != "" {
			values.Prompt = prompt
			values.Suffix = req.Suffix
		} else {
			var msgs []api.Message
			if req.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: req.System})
			} else if m.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: m.System})
			}

Michael Yang's avatar
Michael Yang committed
210
211
212
213
			if req.Context == nil {
				msgs = append(msgs, m.Messages...)
			}

214
215
216
217
218
219
220
			for _, i := range images {
				msgs = append(msgs, api.Message{Role: "user", Content: fmt.Sprintf("[img-%d]", i.ID)})
			}

			values.Messages = append(msgs, api.Message{Role: "user", Content: req.Prompt})
		}

Michael Yang's avatar
Michael Yang committed
221
222
		var b bytes.Buffer
		if req.Context != nil {
223
			s, err := r.Detokenize(c.Request.Context(), req.Context)
Michael Yang's avatar
Michael Yang committed
224
225
226
227
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
228
			b.WriteString(s)
Michael Yang's avatar
Michael Yang committed
229
		}
230
231
232
233
234
235
236

		if err := tmpl.Execute(&b, values); err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		prompt = b.String()
Bruce MacDonald's avatar
Bruce MacDonald committed
237
238
	}

Michael Yang's avatar
Michael Yang committed
239
	slog.Debug("generate request", "prompt", prompt, "images", images)
240

Bruce MacDonald's avatar
Bruce MacDonald committed
241
242
	ch := make(chan any)
	go func() {
243
244
		// TODO (jmorganca): avoid building the response twice both here and below
		var sb strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
245
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
246
		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
Michael Yang's avatar
Michael Yang committed
247
248
249
			Prompt:  prompt,
			Images:  images,
			Format:  req.Format,
Michael Yang's avatar
Michael Yang committed
250
			Options: opts,
251
252
		}, func(cr llm.CompletionResponse) {
			res := api.GenerateResponse{
253
254
				Model:      req.Model,
				CreatedAt:  time.Now().UTC(),
255
256
257
				Response:   cr.Content,
				Done:       cr.Done,
				DoneReason: cr.DoneReason,
Bruce MacDonald's avatar
Bruce MacDonald committed
258
				Metrics: api.Metrics{
259
260
261
262
					PromptEvalCount:    cr.PromptEvalCount,
					PromptEvalDuration: cr.PromptEvalDuration,
					EvalCount:          cr.EvalCount,
					EvalDuration:       cr.EvalDuration,
Bruce MacDonald's avatar
Bruce MacDonald committed
263
				},
Bruce MacDonald's avatar
Bruce MacDonald committed
264
			}
265
266
267
268
269
270
271
272
273
274

			if _, err := sb.WriteString(cr.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
			}

			if cr.Done {
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)

				if !req.Raw {
275
					tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String())
276
277
278
279
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
280
					res.Context = tokens
281
282
283
284
				}
			}

			ch <- res
Michael Yang's avatar
Michael Yang committed
285
		}); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
286
287
288
289
290
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
Michael Yang committed
291
		var r api.GenerateResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
292
		var sb strings.Builder
Michael Yang's avatar
Michael Yang committed
293
294
		for rr := range ch {
			switch t := rr.(type) {
295
			case api.GenerateResponse:
Michael Yang's avatar
Michael Yang committed
296
297
				sb.WriteString(t.Response)
				r = t
298
			case gin.H:
Michael Yang's avatar
Michael Yang committed
299
300
301
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
302
				}
Michael Yang's avatar
Michael Yang committed
303
304
305

				c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
				return
306
			default:
Michael Yang's avatar
Michael Yang committed
307
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
Bruce MacDonald's avatar
Bruce MacDonald committed
308
309
310
				return
			}
		}
311

Michael Yang's avatar
Michael Yang committed
312
313
		r.Response = sb.String()
		c.JSON(http.StatusOK, r)
Bruce MacDonald's avatar
Bruce MacDonald committed
314
315
316
317
318
319
		return
	}

	streamResponse(c, ch)
}

320
func (s *Server) EmbedHandler(c *gin.Context) {
321
	checkpointStart := time.Now()
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
	var req api.EmbedRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	truncate := true

	if req.Truncate != nil && !*req.Truncate {
		truncate = false
	}

	var input []string

	switch i := req.Input.(type) {
	case string:
		if len(i) > 0 {
			input = append(input, i)
		}
	case []any:
		for _, v := range i {
			if _, ok := v.(string); !ok {
				c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
				return
			}
			input = append(input, v.(string))
		}
	default:
355
356
357
358
		if req.Input != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
			return
		}
359
360
361
362
363
364
365
366
	}

	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}

367
368
	checkpointLoaded := time.Now()

369
370
371
372
373
	if len(input) == 0 {
		c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
		return
	}

374
375
376
377
378
379
	kvData, err := getKVData(m.ModelPath, false)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

380
	var count int
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
	for i, s := range input {
		tokens, err := r.Tokenize(c.Request.Context(), s)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		ctxLen := min(opts.NumCtx, int(kvData.ContextLength()))
		if len(tokens) > ctxLen {
			if !truncate {
				c.JSON(http.StatusBadRequest, gin.H{"error": "input length exceeds maximum context length"})
				return
			}

			tokens = tokens[:ctxLen]
			s, err = r.Detokenize(c.Request.Context(), tokens)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

403
404
		count += len(tokens)

405
406
		input[i] = s
	}
407
408
409
410
411
412
413
414
415
416
417
418

	var g errgroup.Group
	embeddings := make([][]float32, len(input))
	for i, text := range input {
		g.Go(func() error {
			embedding, err := r.Embedding(c.Request.Context(), text)
			if err != nil {
				return err
			}
			embeddings[i] = normalize(embedding)
			return nil
		})
419
420
	}

421
422
423
424
	if err := g.Wait(); err != nil {
		slog.Error("embedding generation failed", "error", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Errorf("failed to generate embeddings: %v", err)})
		return
425
426
427
	}

	resp := api.EmbedResponse{
428
		Model:           req.Model,
429
		Embeddings:      embeddings,
430
431
		TotalDuration:   time.Since(checkpointStart),
		LoadDuration:    checkpointLoaded.Sub(checkpointStart),
432
		PromptEvalCount: count,
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
	}
	c.JSON(http.StatusOK, resp)
}

func normalize(vec []float32) []float32 {
	var sum float32
	for _, v := range vec {
		sum += v * v
	}

	norm := float32(0.0)
	if sum > 0 {
		norm = float32(1.0 / math.Sqrt(float64(sum)))
	}

	for i := range vec {
		vec[i] *= norm
	}
	return vec
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
454
func (s *Server) EmbeddingsHandler(c *gin.Context) {
Bruce MacDonald's avatar
Bruce MacDonald committed
455
	var req api.EmbeddingRequest
Michael Yang's avatar
Michael Yang committed
456
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
457
458
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
459
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
460
461
462
463
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
464
	r, _, _, err := s.scheduleRunner(c.Request.Context(), req.Model, []Capability{}, req.Options, req.KeepAlive)
Bruce MacDonald's avatar
Bruce MacDonald committed
465
	if err != nil {
Michael Yang's avatar
Michael Yang committed
466
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
467
468
469
		return
	}

470
471
472
	// an empty request loads the model
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float64{}})
Bruce MacDonald's avatar
Bruce MacDonald committed
473
474
475
		return
	}

476
	embedding, err := r.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
477
	if err != nil {
478
		slog.Info(fmt.Sprintf("embedding generation failed: %v", err))
Bruce MacDonald's avatar
Bruce MacDonald committed
479
480
481
482
		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
		return
	}

483
484
485
	var e []float64
	for _, v := range embedding {
		e = append(e, float64(v))
486
487
488
	}

	resp := api.EmbeddingResponse{
489
		Embedding: e,
490
491
	}
	c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
492
493
}

494
func (s *Server) PullHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
495
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
496
497
498
499
500
501
502
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
503
504
505
		return
	}

506
507
508
509
510
511
512
513
	name := model.ParseName(cmp.Or(req.Model, req.Name))
	if !name.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid model name"})
		return
	}

	if err := checkNameExists(name); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
514
515
516
		return
	}

517
518
519
	ch := make(chan any)
	go func() {
		defer close(ch)
520
521
		fn := func(r api.ProgressResponse) {
			ch <- r
522
		}
523

Michael Yang's avatar
Michael Yang committed
524
		regOpts := &registryOptions{
525
526
527
			Insecure: req.Insecure,
		}

528
529
530
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

531
		if err := PullModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
532
			ch <- gin.H{"error": err.Error()}
533
534
535
		}
	}()

536
537
538
539
540
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

541
542
543
	streamResponse(c, ch)
}

544
func (s *Server) PushHandler(c *gin.Context) {
545
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
546
547
548
549
550
551
552
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
553
554
		return
	}
Michael Yang's avatar
Michael Yang committed
555

Michael Yang's avatar
Michael Yang committed
556
557
558
559
560
561
562
	var model string
	if req.Model != "" {
		model = req.Model
	} else if req.Name != "" {
		model = req.Name
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
563
564
565
		return
	}

566
567
568
	ch := make(chan any)
	go func() {
		defer close(ch)
569
570
		fn := func(r api.ProgressResponse) {
			ch <- r
571
		}
572

Michael Yang's avatar
Michael Yang committed
573
		regOpts := &registryOptions{
574
575
576
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
577
578
579
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

Michael Yang's avatar
Michael Yang committed
580
		if err := PushModel(ctx, model, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
581
			ch <- gin.H{"error": err.Error()}
582
583
584
		}
	}()

585
586
587
588
589
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

590
591
592
	streamResponse(c, ch)
}

593
594
595
596
597
598
599
600
func checkNameExists(name model.Name) error {
	names, err := Manifests()
	if err != nil {
		return err
	}

	for n := range names {
		if strings.EqualFold(n.Filepath(), name.Filepath()) && n != name {
Michael Yang's avatar
lint  
Michael Yang committed
601
			return errors.New("a model with that name already exists")
602
603
604
605
606
607
		}
	}

	return nil
}

608
func (s *Server) CreateHandler(c *gin.Context) {
609
610
	var r api.CreateRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
611
612
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
613
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
614
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
615
		return
616
617
	}

618
	name := model.ParseName(cmp.Or(r.Model, r.Name))
Michael Yang's avatar
Michael Yang committed
619
	if !name.IsValid() {
620
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": errtypes.InvalidModelNameErrMsg})
621
622
623
		return
	}

624
625
626
627
628
	if err := checkNameExists(name); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

629
	if r.Path == "" && r.Modelfile == "" {
Michael Yang's avatar
Michael Yang committed
630
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"})
Michael Yang's avatar
Michael Yang committed
631
632
		return
	}
Michael Yang's avatar
Michael Yang committed
633

634
635
636
	var sr io.Reader = strings.NewReader(r.Modelfile)
	if r.Path != "" && r.Modelfile == "" {
		f, err := os.Open(r.Path)
Michael Yang's avatar
Michael Yang committed
637
638
639
640
		if err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)})
			return
		}
Michael Yang's avatar
Michael Yang committed
641
		defer f.Close()
Michael Yang's avatar
Michael Yang committed
642

643
		sr = f
Michael Yang's avatar
Michael Yang committed
644
	}
Michael Yang's avatar
Michael Yang committed
645

646
	f, err := parser.ParseFile(sr)
Michael Yang's avatar
Michael Yang committed
647
648
649
650
651
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
652
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
653
654
	go func() {
		defer close(ch)
655
656
		fn := func(resp api.ProgressResponse) {
			ch <- resp
657
658
		}

659
660
661
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

662
		quantization := cmp.Or(r.Quantize, r.Quantization)
Josh's avatar
Josh committed
663
664
665
		if err := CreateModel(ctx, name, filepath.Dir(r.Path), strings.ToUpper(quantization), f, fn); errors.Is(err, errBadTemplate) {
			ch <- gin.H{"error": err.Error(), "status": http.StatusBadRequest}
		} else if err != nil {
Michael Yang's avatar
Michael Yang committed
666
			ch <- gin.H{"error": err.Error()}
667
		}
Michael Yang's avatar
Michael Yang committed
668
	}()
Michael Yang's avatar
Michael Yang committed
669

670
	if r.Stream != nil && !*r.Stream {
671
672
673
674
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
675
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
676
677
}

678
func (s *Server) DeleteHandler(c *gin.Context) {
679
680
	var r api.DeleteRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
681
682
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
683
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
684
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
685
686
687
		return
	}

688
689
690
	n := model.ParseName(cmp.Or(r.Model, r.Name))
	if !n.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("name %q is invalid", cmp.Or(r.Model, r.Name))})
691
692
		return
	}
Michael Yang's avatar
Michael Yang committed
693

694
	m, err := ParseNamedManifest(n)
Michael Yang's avatar
Michael Yang committed
695
696
697
698
699
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

700
	if err := m.Remove(); err != nil {
Michael Yang's avatar
Michael Yang committed
701
702
703
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
704
705
706
707
708

	if err := m.RemoveLayers(); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
709
710
}

711
func (s *Server) ShowHandler(c *gin.Context) {
Patrick Devine's avatar
Patrick Devine committed
712
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
713
714
715
716
717
718
719
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
720
721
722
		return
	}

Michael Yang's avatar
Michael Yang committed
723
	if req.Model != "" {
Michael Yang's avatar
Michael Yang committed
724
		// noop
Michael Yang's avatar
Michael Yang committed
725
	} else if req.Name != "" {
Michael Yang's avatar
Michael Yang committed
726
		req.Model = req.Name
Michael Yang's avatar
Michael Yang committed
727
	} else {
728
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
729
730
731
		return
	}

732
	resp, err := GetModelInfo(req)
Patrick Devine's avatar
Patrick Devine committed
733
	if err != nil {
734
735
		switch {
		case os.IsNotExist(err):
Michael Yang's avatar
Michael Yang committed
736
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
737
738
739
		case err.Error() == "invalid model name":
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
Patrick Devine's avatar
Patrick Devine committed
740
741
742
743
744
745
746
747
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

748
func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
749
	m, err := GetModel(req.Model)
Patrick Devine's avatar
Patrick Devine committed
750
751
752
753
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
754
	modelDetails := api.ModelDetails{
755
756
757
758
759
760
		ParentModel:       m.ParentModel,
		Format:            m.Config.ModelFormat,
		Family:            m.Config.ModelFamily,
		Families:          m.Config.ModelFamilies,
		ParameterSize:     m.Config.ModelType,
		QuantizationLevel: m.Config.FileType,
Patrick Devine's avatar
Patrick Devine committed
761
762
	}

763
	if req.System != "" {
764
		m.System = req.System
765
766
	}

Michael Yang's avatar
Michael Yang committed
767
768
769
	msgs := make([]api.Message, len(m.Messages))
	for i, msg := range m.Messages {
		msgs[i] = api.Message{Role: msg.Role, Content: msg.Content}
770
771
	}

772
773
	n := model.ParseName(req.Model)
	if !n.IsValid() {
Michael Yang's avatar
lint  
Michael Yang committed
774
		return nil, errors.New("invalid model name")
775
776
777
778
779
780
781
	}

	manifest, err := ParseNamedManifest(n)
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
782
	resp := &api.ShowResponse{
783
784
		License:    strings.Join(m.License, "\n"),
		System:     m.System,
Michael Yang's avatar
Michael Yang committed
785
		Template:   m.Template.String(),
786
787
788
		Details:    modelDetails,
		Messages:   msgs,
		ModifiedAt: manifest.fi.ModTime(),
Patrick Devine's avatar
Patrick Devine committed
789
790
791
792
	}

	var params []string
	cs := 30
793
	for k, v := range m.Options {
Patrick Devine's avatar
Patrick Devine committed
794
795
796
		switch val := v.(type) {
		case []interface{}:
			for _, nv := range val {
Patrick Devine's avatar
Patrick Devine committed
797
				params = append(params, fmt.Sprintf("%-*s %#v", cs, k, nv))
Patrick Devine's avatar
Patrick Devine committed
798
			}
Patrick Devine's avatar
Patrick Devine committed
799
800
		default:
			params = append(params, fmt.Sprintf("%-*s %#v", cs, k, v))
Patrick Devine's avatar
Patrick Devine committed
801
802
803
804
		}
	}
	resp.Parameters = strings.Join(params, "\n")

805
806
	for k, v := range req.Options {
		if _, ok := req.Options[k]; ok {
807
			m.Options[k] = v
808
809
810
		}
	}

811
	var sb strings.Builder
812
	fmt.Fprintln(&sb, "# Modelfile generated by \"ollama show\"")
813
	fmt.Fprintln(&sb, "# To build a new Modelfile based on this, replace FROM with:")
814
815
	fmt.Fprintf(&sb, "# FROM %s\n\n", m.ShortName)
	fmt.Fprint(&sb, m.String())
816
	resp.Modelfile = sb.String()
817

818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
	kvData, err := getKVData(m.ModelPath, req.Verbose)
	if err != nil {
		return nil, err
	}
	delete(kvData, "general.name")
	delete(kvData, "tokenizer.chat_template")
	resp.ModelInfo = kvData

	if len(m.ProjectorPaths) > 0 {
		projectorData, err := getKVData(m.ProjectorPaths[0], req.Verbose)
		if err != nil {
			return nil, err
		}
		resp.ProjectorInfo = projectorData
	}

Patrick Devine's avatar
Patrick Devine committed
834
835
836
	return resp, nil
}

837
func getKVData(digest string, verbose bool) (llm.KV, error) {
838
839
840
841
842
	maxArraySize := 0
	if verbose {
		maxArraySize = -1
	}
	kvData, err := llm.LoadModel(digest, maxArraySize)
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
	if err != nil {
		return nil, err
	}

	kv := kvData.KV()

	if !verbose {
		for k := range kv {
			if t, ok := kv[k].([]any); len(t) > 5 && ok {
				kv[k] = []any{}
			}
		}
	}

	return kv, nil
}

860
func (s *Server) ListHandler(c *gin.Context) {
861
	ms, err := Manifests()
Patrick Devine's avatar
Patrick Devine committed
862
863
864
865
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
866

867
	models := []api.ListModelResponse{}
868
869
	for n, m := range ms {
		var cf ConfigV2
870
871
872
873
874
875
876
877
878
879
880
881
882

		if m.Config.Digest != "" {
			f, err := m.Config.Open()
			if err != nil {
				slog.Warn("bad manifest filepath", "name", n, "error", err)
				continue
			}
			defer f.Close()

			if err := json.NewDecoder(f).Decode(&cf); err != nil {
				slog.Warn("bad manifest config", "name", n, "error", err)
				continue
			}
Patrick Devine's avatar
Patrick Devine committed
883
		}
Michael Yang's avatar
Michael Yang committed
884

885
		// tag should never be masked
886
		models = append(models, api.ListModelResponse{
887
888
889
890
891
892
893
894
895
896
897
898
899
			Model:      n.DisplayShortest(),
			Name:       n.DisplayShortest(),
			Size:       m.Size(),
			Digest:     m.digest,
			ModifiedAt: m.fi.ModTime(),
			Details: api.ModelDetails{
				Format:            cf.ModelFormat,
				Family:            cf.ModelFamily,
				Families:          cf.ModelFamilies,
				ParameterSize:     cf.ModelType,
				QuantizationLevel: cf.FileType,
			},
		})
Patrick Devine's avatar
Patrick Devine committed
900
901
	}

902
	slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
903
904
905
906
		// most recently modified first
		return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
	})

Michael Yang's avatar
Michael Yang committed
907
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
908
909
}

910
func (s *Server) CopyHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
911
912
	var r api.CopyRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
913
914
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
915
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
916
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
917
918
919
		return
	}

Michael Yang's avatar
Michael Yang committed
920
921
	src := model.ParseName(r.Source)
	if !src.IsValid() {
Michael Yang's avatar
Michael Yang committed
922
923
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("source %q is invalid", r.Source)})
		return
924
925
	}

Michael Yang's avatar
Michael Yang committed
926
927
	dst := model.ParseName(r.Destination)
	if !dst.IsValid() {
928
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("destination %q is invalid", r.Destination)})
Patrick Devine's avatar
Patrick Devine committed
929
930
		return
	}
Michael Yang's avatar
Michael Yang committed
931

932
933
934
935
936
	if err := checkNameExists(dst); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
937
938
939
940
941
	if err := CopyModel(src, dst); errors.Is(err, os.ErrNotExist) {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
	} else if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
	}
Patrick Devine's avatar
Patrick Devine committed
942
943
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
944
func (s *Server) HeadBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
945
946
947
948
949
950
951
952
953
954
955
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); err != nil {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	}

Michael Yang's avatar
Michael Yang committed
956
	c.Status(http.StatusOK)
Michael Yang's avatar
Michael Yang committed
957
958
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
959
func (s *Server) CreateBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
960
961
	if ib, ok := intermediateBlobs[c.Param("digest")]; ok {
		p, err := GetBlobsPath(ib)
962
963
964
965
966
967
		if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
Michael Yang's avatar
Michael Yang committed
968
969
			slog.Info("evicting intermediate blob which no longer exists", "digest", ib)
			delete(intermediateBlobs, c.Param("digest"))
970
971
972
973
974
975
976
977
978
		} else if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		} else {
			c.Status(http.StatusOK)
			return
		}
	}

979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	_, err = os.Stat(path)
	switch {
	case errors.Is(err, os.ErrNotExist):
		// noop
	case err != nil:
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	default:
		c.Status(http.StatusOK)
		return
	}

997
	layer, err := NewLayer(c.Request.Body, "")
Michael Yang's avatar
Michael Yang committed
998
999
1000
1001
1002
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

1003
1004
	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
Michael Yang's avatar
Michael Yang committed
1005
1006
1007
		return
	}

Michael Yang's avatar
Michael Yang committed
1008
	c.Status(http.StatusCreated)
Michael Yang's avatar
Michael Yang committed
1009
1010
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
func isLocalIP(ip netip.Addr) bool {
	if interfaces, err := net.Interfaces(); err == nil {
		for _, iface := range interfaces {
			addrs, err := iface.Addrs()
			if err != nil {
				continue
			}

			for _, a := range addrs {
				if parsed, _, err := net.ParseCIDR(a.String()); err == nil {
					if parsed.String() == ip.String() {
						return true
					}
				}
			}
		}
	}

	return false
}

1032
func allowedHost(host string) bool {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1033
	if host == "" || host == "localhost" {
1034
1035
1036
1037
1038
1039
1040
		return true
	}

	if hostname, err := os.Hostname(); err == nil && host == hostname {
		return true
	}

Michael Yang's avatar
lint  
Michael Yang committed
1041
	tlds := []string{
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1042
1043
1044
		"localhost",
		"local",
		"internal",
1045
	}
1046

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1047
	// check if the host is a local TLD
1048
1049
1050
1051
1052
1053
	for _, tld := range tlds {
		if strings.HasSuffix(host, "."+tld) {
			return true
		}
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1054
	return false
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1055
}
1056

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1057
1058
1059
func allowedHostsMiddleware(addr net.Addr) gin.HandlerFunc {
	return func(c *gin.Context) {
		if addr == nil {
1060
1061
1062
1063
			c.Next()
			return
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1064
		if addr, err := netip.ParseAddrPort(addr.String()); err == nil && !addr.Addr().IsLoopback() {
1065
1066
1067
1068
1069
1070
1071
1072
1073
			c.Next()
			return
		}

		host, _, err := net.SplitHostPort(c.Request.Host)
		if err != nil {
			host = c.Request.Host
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1074
		if addr, err := netip.ParseAddr(host); err == nil {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1075
			if addr.IsLoopback() || addr.IsPrivate() || addr.IsUnspecified() || isLocalIP(addr) {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1076
1077
1078
1079
1080
				c.Next()
				return
			}
		}

1081
		if allowedHost(host) {
Michael Yang's avatar
lint  
Michael Yang committed
1082
			if c.Request.Method == http.MethodOptions {
1083
1084
1085
1086
				c.AbortWithStatus(http.StatusNoContent)
				return
			}

1087
1088
1089
1090
1091
1092
			c.Next()
			return
		}

		c.AbortWithStatus(http.StatusForbidden)
	}
1093
}
1094

1095
func (s *Server) GenerateRoutes() http.Handler {
Michael Yang's avatar
Michael Yang committed
1096
1097
	config := cors.DefaultConfig()
	config.AllowWildcard = true
1098
	config.AllowBrowserExtensions = true
1099
	config.AllowHeaders = []string{"Authorization", "Content-Type", "User-Agent", "Accept", "X-Requested-With"}
royjhan's avatar
royjhan committed
1100
1101
1102
1103
	openAIProperties := []string{"lang", "package-version", "os", "arch", "runtime", "runtime-version", "async"}
	for _, prop := range openAIProperties {
		config.AllowHeaders = append(config.AllowHeaders, "x-stainless-"+prop)
	}
Michael Yang's avatar
origins  
Michael Yang committed
1104
	config.AllowOrigins = envconfig.Origins()
Michael Yang's avatar
Michael Yang committed
1105

Bruce MacDonald's avatar
Bruce MacDonald committed
1106
	r := gin.Default()
1107
1108
	r.Use(
		cors.New(config),
1109
		allowedHostsMiddleware(s.addr),
1110
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
1111

1112
	r.POST("/api/pull", s.PullHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1113
1114
	r.POST("/api/generate", s.GenerateHandler)
	r.POST("/api/chat", s.ChatHandler)
1115
	r.POST("/api/embed", s.EmbedHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1116
	r.POST("/api/embeddings", s.EmbeddingsHandler)
1117
1118
1119
1120
1121
	r.POST("/api/create", s.CreateHandler)
	r.POST("/api/push", s.PushHandler)
	r.POST("/api/copy", s.CopyHandler)
	r.DELETE("/api/delete", s.DeleteHandler)
	r.POST("/api/show", s.ShowHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1122
1123
	r.POST("/api/blobs/:digest", s.CreateBlobHandler)
	r.HEAD("/api/blobs/:digest", s.HeadBlobHandler)
1124
	r.GET("/api/ps", s.PsHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1125

1126
	// Compatibility endpoints
1127
	r.POST("/v1/chat/completions", openai.ChatMiddleware(), s.ChatHandler)
1128
	r.POST("/v1/completions", openai.CompletionsMiddleware(), s.GenerateHandler)
1129
	r.POST("/v1/embeddings", openai.EmbeddingsMiddleware(), s.EmbedHandler)
1130
1131
	r.GET("/v1/models", openai.ListMiddleware(), s.ListHandler)
	r.GET("/v1/models/:model", openai.RetrieveMiddleware(), s.ShowHandler)
1132

Michael Yang's avatar
Michael Yang committed
1133
1134
1135
1136
1137
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

1138
		r.Handle(method, "/api/tags", s.ListHandler)
Michael Yang's avatar
Michael Yang committed
1139
1140
1141
		r.Handle(method, "/api/version", func(c *gin.Context) {
			c.JSON(http.StatusOK, gin.H{"version": version.Version})
		})
Michael Yang's avatar
Michael Yang committed
1142
1143
	}

1144
1145
1146
1147
	return r
}

func Serve(ln net.Listener) error {
Michael Yang's avatar
Michael Yang committed
1148
	level := slog.LevelInfo
Michael Yang's avatar
Michael Yang committed
1149
	if envconfig.Debug() {
Michael Yang's avatar
Michael Yang committed
1150
		level = slog.LevelDebug
1151
	}
Michael Yang's avatar
Michael Yang committed
1152

1153
	slog.Info("server config", "env", envconfig.Values())
Michael Yang's avatar
Michael Yang committed
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
	handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
		Level:     level,
		AddSource: true,
		ReplaceAttr: func(_ []string, attr slog.Attr) slog.Attr {
			if attr.Key == slog.SourceKey {
				source := attr.Value.Any().(*slog.Source)
				source.File = filepath.Base(source.File)
			}

			return attr
		},
	})

	slog.SetDefault(slog.New(handler))

1169
1170
1171
1172
1173
1174
1175
1176
	blobsDir, err := GetBlobsPath("")
	if err != nil {
		return err
	}
	if err := fixBlobs(blobsDir); err != nil {
		return err
	}

Michael Yang's avatar
bool  
Michael Yang committed
1177
	if !envconfig.NoPrune() {
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
		// clean up unused layers and manifests
		if err := PruneLayers(); err != nil {
			return err
		}

		manifestsPath, err := GetManifestPath()
		if err != nil {
			return err
		}

		if err := PruneDirectory(manifestsPath); err != nil {
			return err
		}
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1193
	ctx, done := context.WithCancel(context.Background())
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1194
1195
	schedCtx, schedDone := context.WithCancel(ctx)
	sched := InitScheduler(schedCtx)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1196
	s := &Server{addr: ln.Addr(), sched: sched}
1197
1198

	http.Handle("/", s.GenerateRoutes())
1199

1200
	slog.Info(fmt.Sprintf("Listening on %s (version %s)", ln.Addr(), version.Version))
1201
	srvr := &http.Server{
1202
1203
1204
1205
1206
1207
1208
1209
1210
		// Use http.DefaultServeMux so we get net/http/pprof for
		// free.
		//
		// TODO(bmizerany): Decide if we want to make this
		// configurable so it is not exposed by default, or allow
		// users to bind it to a different port. This was a quick
		// and easy way to get pprof, but it may not be the best
		// way.
		Handler: nil,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1211
1212
	}

1213
1214
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
1215
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
1216
1217
	go func() {
		<-signals
1218
		srvr.Close()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1219
		schedDone()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1220
		sched.unloadAllRunners()
1221
		runners.Cleanup(build.EmbedFS)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1222
		done()
1223
1224
	}()

1225
1226
	if _, err := runners.Refresh(build.EmbedFS); err != nil {
		return fmt.Errorf("unable to initialize llm runners %w", err)
1227
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1228

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1229
	s.sched.Run(schedCtx)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1230
1231
1232

	// At startup we retrieve GPU information so we can get log messages before loading a model
	// This will log warnings to the log in case we have problems with detected GPUs
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1233
1234
	gpus := gpu.GetGPUInfo()
	gpus.LogDetails()
1235

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1236
1237
1238
1239
1240
1241
1242
	err = srvr.Serve(ln)
	// If server is closed from the signal handler, wait for the ctx to be done
	// otherwise error out quickly
	if !errors.Is(err, http.ErrServerClosed) {
		return err
	}
	<-ctx.Done()
1243
	return nil
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1244
}
Michael Yang's avatar
Michael Yang committed
1245

1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
Josh's avatar
Josh committed
1256
1257
1258
1259
			status, ok := r["status"].(int)
			if !ok {
				status = http.StatusInternalServerError
			}
1260
			if errorMsg, ok := r["error"].(string); ok {
Josh's avatar
Josh committed
1261
				c.JSON(status, gin.H{"error": errorMsg})
1262
1263
				return
			} else {
Josh's avatar
Josh committed
1264
				c.JSON(status, gin.H{"error": "unexpected error format in progress response"})
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
1275
func streamResponse(c *gin.Context, ch chan any) {
1276
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
1277
1278
1279
1280
1281
1282
1283
1284
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
1285
			slog.Info(fmt.Sprintf("streamResponse: json.Marshal failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1286
1287
1288
			return false
		}

1289
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
1290
1291
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
1292
			slog.Info(fmt.Sprintf("streamResponse: w.Write failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1293
1294
1295
1296
1297
1298
			return false
		}

		return true
	})
}
Bruce MacDonald's avatar
Bruce MacDonald committed
1299

1300
func (s *Server) PsHandler(c *gin.Context) {
1301
	models := []api.ProcessModelResponse{}
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312

	for _, v := range s.sched.loaded {
		model := v.model
		modelDetails := api.ModelDetails{
			Format:            model.Config.ModelFormat,
			Family:            model.Config.ModelFamily,
			Families:          model.Config.ModelFamilies,
			ParameterSize:     model.Config.ModelType,
			QuantizationLevel: model.Config.FileType,
		}

1313
		mr := api.ProcessModelResponse{
1314
1315
1316
1317
1318
1319
1320
1321
			Model:     model.ShortName,
			Name:      model.ShortName,
			Size:      int64(v.estimatedTotal),
			SizeVRAM:  int64(v.estimatedVRAM),
			Digest:    model.Digest,
			Details:   modelDetails,
			ExpiresAt: v.expiresAt,
		}
1322
1323
1324
1325
1326
1327
1328
1329
		// The scheduler waits to set expiresAt, so if a model is loading it's
		// possible that it will be set to the unix epoch. For those cases, just
		// calculate the time w/ the sessionDuration instead.
		var epoch time.Time
		if v.expiresAt == epoch {
			mr.ExpiresAt = time.Now().Add(v.sessionDuration)
		}

1330
1331
1332
		models = append(models, mr)
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1333
1334
1335
1336
1337
	slices.SortStableFunc(models, func(i, j api.ProcessModelResponse) int {
		// longest duration remaining listed first
		return cmp.Compare(j.ExpiresAt.Unix(), i.ExpiresAt.Unix())
	})

1338
	c.JSON(http.StatusOK, api.ProcessResponse{Models: models})
1339
1340
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1341
func (s *Server) ChatHandler(c *gin.Context) {
1342
1343
	checkpointStart := time.Now()

Bruce MacDonald's avatar
Bruce MacDonald committed
1344
	var req api.ChatRequest
Michael Yang's avatar
Michael Yang committed
1345
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1346
1347
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
1348
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1349
1350
1351
1352
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
	// expire the runner
	if len(req.Messages) == 0 && req.KeepAlive != nil && int(req.KeepAlive.Seconds()) == 0 {
		model, err := GetModel(req.Model)
		if err != nil {
			switch {
			case os.IsNotExist(err):
				c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
			case err.Error() == "invalid model name":
				c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			}
			return
		}
		s.sched.expireRunner(model)

		c.JSON(http.StatusOK, api.ChatResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

Michael Yang's avatar
Michael Yang committed
1379
	caps := []Capability{CapabilityCompletion}
1380
	if len(req.Tools) > 0 {
Michael Yang's avatar
tools  
Michael Yang committed
1381
1382
1383
		caps = append(caps, CapabilityTools)
	}

Michael Yang's avatar
Michael Yang committed
1384
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), req.Model, caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
1385
1386
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support chat", req.Model)})
Bruce MacDonald's avatar
Bruce MacDonald committed
1387
		return
Michael Yang's avatar
Michael Yang committed
1388
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
1389
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
1390
1391
		return
	}
Michael Yang's avatar
Michael Yang committed
1392

1393
1394
	checkpointLoaded := time.Now()

Michael Yang's avatar
Michael Yang committed
1395
1396
	if len(req.Messages) == 0 {
		c.JSON(http.StatusOK, api.ChatResponse{
1397
			Model:      req.Model,
Michael Yang's avatar
Michael Yang committed
1398
1399
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
1400
1401
			Done:       true,
			DoneReason: "load",
Michael Yang's avatar
Michael Yang committed
1402
		})
1403
1404
1405
		return
	}

Michael Yang's avatar
Michael Yang committed
1406
	msgs := append(m.Messages, req.Messages...)
1407
	if req.Messages[0].Role != "system" && m.System != "" {
Michael Yang's avatar
Michael Yang committed
1408
		msgs = append([]api.Message{{Role: "system", Content: m.System}}, msgs...)
1409
1410
	}

Michael Yang's avatar
Michael Yang committed
1411
	prompt, images, err := chatPrompt(c.Request.Context(), m, r.Tokenize, opts, msgs, req.Tools)
Michael Yang's avatar
Michael Yang committed
1412
1413
1414
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
1415
1416
	}

Michael Yang's avatar
Michael Yang committed
1417
	slog.Debug("chat request", "images", len(images), "prompt", prompt)
1418

Bruce MacDonald's avatar
Bruce MacDonald committed
1419
1420
1421
	ch := make(chan any)
	go func() {
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
1422
		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
Michael Yang's avatar
Michael Yang committed
1423
1424
1425
			Prompt:  prompt,
			Images:  images,
			Format:  req.Format,
Michael Yang's avatar
Michael Yang committed
1426
			Options: opts,
Michael Yang's avatar
Michael Yang committed
1427
		}, func(r llm.CompletionResponse) {
1428
			res := api.ChatResponse{
1429
1430
1431
1432
1433
				Model:      req.Model,
				CreatedAt:  time.Now().UTC(),
				Message:    api.Message{Role: "assistant", Content: r.Content},
				Done:       r.Done,
				DoneReason: r.DoneReason,
Bruce MacDonald's avatar
Bruce MacDonald committed
1434
1435
1436
1437
1438
1439
1440
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
			}
1441
1442
1443
1444
1445
1446
1447

			if r.Done {
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
			}

			ch <- res
Michael Yang's avatar
Michael Yang committed
1448
		}); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1449
1450
1451
1452
1453
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
tools  
Michael Yang committed
1454
		var resp api.ChatResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
1455
		var sb strings.Builder
Michael Yang's avatar
Michael Yang committed
1456
1457
		for rr := range ch {
			switch t := rr.(type) {
1458
			case api.ChatResponse:
Michael Yang's avatar
Michael Yang committed
1459
				sb.WriteString(t.Message.Content)
Michael Yang's avatar
tools  
Michael Yang committed
1460
				resp = t
1461
			case gin.H:
Michael Yang's avatar
Michael Yang committed
1462
1463
1464
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
1465
				}
Michael Yang's avatar
Michael Yang committed
1466
1467
1468

				c.JSON(http.StatusInternalServerError, gin.H{"error": msg})
				return
1469
			default:
Michael Yang's avatar
Michael Yang committed
1470
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
1471
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
1472
1473
			}
		}
1474

Michael Yang's avatar
tools  
Michael Yang committed
1475
		resp.Message.Content = sb.String()
1476
1477
1478
1479
1480
1481

		if len(req.Tools) > 0 {
			if toolCalls, ok := m.parseToolCalls(sb.String()); ok {
				resp.Message.ToolCalls = toolCalls
				resp.Message.Content = ""
			}
Michael Yang's avatar
tools  
Michael Yang committed
1482
1483
1484
		}

		c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
1485
1486
1487
1488
1489
		return
	}

	streamResponse(c, ch)
}
1490

Michael Yang's avatar
Michael Yang committed
1491
func handleScheduleError(c *gin.Context, name string, err error) {
Michael Yang's avatar
Michael Yang committed
1492
	switch {
1493
	case errors.Is(err, errCapabilities), errors.Is(err, errRequired):
Michael Yang's avatar
Michael Yang committed
1494
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
1495
	case errors.Is(err, context.Canceled):
1496
		c.JSON(499, gin.H{"error": "request canceled"})
Michael Yang's avatar
Michael Yang committed
1497
	case errors.Is(err, ErrMaxQueue):
1498
		c.JSON(http.StatusServiceUnavailable, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
1499
1500
	case errors.Is(err, os.ErrNotExist):
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found, try pulling it first", name)})
Michael Yang's avatar
Michael Yang committed
1501
1502
	default:
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
1503
1504
	}
}