routes.go 18.7 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
4
	"context"
Michael Yang's avatar
Michael Yang committed
5
	"encoding/json"
6
	"errors"
7
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
8
	"io"
9
	"io/fs"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
10
11
12
	"log"
	"net"
	"net/http"
13
	"os"
14
	"os/signal"
Michael Yang's avatar
Michael Yang committed
15
	"path/filepath"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
16
	"reflect"
17
	"runtime"
Patrick Devine's avatar
Patrick Devine committed
18
	"strconv"
Michael Yang's avatar
Michael Yang committed
19
	"strings"
Michael Yang's avatar
Michael Yang committed
20
	"sync"
21
	"syscall"
22
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
23

Michael Yang's avatar
Michael Yang committed
24
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
25
26
	"github.com/gin-gonic/gin"

Jeffrey Morgan's avatar
Jeffrey Morgan committed
27
	"github.com/jmorganca/ollama/api"
28
	"github.com/jmorganca/ollama/llm"
Michael Yang's avatar
Michael Yang committed
29
	"github.com/jmorganca/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
30
31
)

Michael Yang's avatar
Michael Yang committed
32
33
34
35
36
37
38
39
40
41
42
43
44
45
var mode string = gin.DebugMode

func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
46
var loaded struct {
Michael Yang's avatar
Michael Yang committed
47
48
	mu sync.Mutex

49
	runner llm.LLM
Michael Yang's avatar
Michael Yang committed
50
51
52

	expireAt    time.Time
	expireTimer *time.Timer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
53

54
55
	*Model
	*api.Options
Michael Yang's avatar
Michael Yang committed
56
57
}

58
59
var defaultSessionDuration = 5 * time.Minute

Bruce MacDonald's avatar
Bruce MacDonald committed
60
// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
61
func load(ctx context.Context, workDir string, model *Model, reqOpts map[string]interface{}, sessionDuration time.Duration) error {
62
63
64
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		log.Printf("could not load model options: %v", err)
Bruce MacDonald's avatar
Bruce MacDonald committed
65
		return err
66
67
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
68
69
	if err := opts.FromMap(reqOpts); err != nil {
		return err
70
71
	}

72
	// check if the loaded model is still running in a subprocess, in case something unexpected happened
73
74
	if loaded.runner != nil {
		if err := loaded.runner.Ping(ctx); err != nil {
75
76
			log.Print("loaded llm process not responding, closing now")
			// the subprocess is no longer running, so close it
77
78
79
80
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
81
82
83
		}
	}

84
85
86
87
88
89
90
	needLoad := loaded.runner == nil || // is there a model loaded?
		loaded.ModelPath != model.ModelPath || // has the base model changed?
		!reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
		!reflect.DeepEqual(loaded.Options.Runner, opts.Runner) // have the runner options changed?

	if needLoad {
		if loaded.runner != nil {
91
			log.Println("changing loaded model")
92
93
94
95
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
96
		}
Michael Yang's avatar
Michael Yang committed
97

98
		llmRunner, err := llm.New(workDir, model.ModelPath, model.AdapterPaths, opts)
Michael Yang's avatar
Michael Yang committed
99
		if err != nil {
100
101
102
103
104
105
106
			// some older models are not compatible with newer versions of llama.cpp
			// show a generalized compatibility error until there is a better way to
			// check for model compatibility
			if strings.Contains(err.Error(), "failed to load model") {
				err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
			}

Bruce MacDonald's avatar
Bruce MacDonald committed
107
			return err
Michael Yang's avatar
Michael Yang committed
108
109
		}

110
111
112
		loaded.Model = model
		loaded.runner = llmRunner
		loaded.Options = &opts
Michael Yang's avatar
Michael Yang committed
113
	}
114

Michael Yang's avatar
Michael Yang committed
115
116
117
118
	// update options for the loaded llm
	// TODO(mxyng): this isn't thread safe, but it should be fine for now
	loaded.runner.SetOptions(opts)

Jeffrey Morgan's avatar
Jeffrey Morgan committed
119
	loaded.expireAt = time.Now().Add(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
120

Jeffrey Morgan's avatar
Jeffrey Morgan committed
121
122
123
124
	if loaded.expireTimer == nil {
		loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
			loaded.mu.Lock()
			defer loaded.mu.Unlock()
Michael Yang's avatar
Michael Yang committed
125

Jeffrey Morgan's avatar
Jeffrey Morgan committed
126
			if time.Now().Before(loaded.expireAt) {
Michael Yang's avatar
Michael Yang committed
127
128
129
				return
			}

130
131
			if loaded.runner != nil {
				loaded.runner.Close()
Michael Yang's avatar
Michael Yang committed
132
133
			}

134
135
136
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
137
		})
Michael Yang's avatar
Michael Yang committed
138
	}
139

Jeffrey Morgan's avatar
Jeffrey Morgan committed
140
	loaded.expireTimer.Reset(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
141
142
143
144
145
146
147
148
149
150
	return nil
}

func GenerateHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()

	var req api.GenerateRequest
Michael Yang's avatar
Michael Yang committed
151
152
153
154
155
156
157
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
158
159
160
		return
	}

161
162
163
	// validate the request
	switch {
	case req.Model == "":
164
165
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
166
167
168
	case len(req.Format) > 0 && req.Format != "json":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
		return
169
170
171
	case req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
		return
172
173
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
174
175
	model, err := GetModel(req.Model)
	if err != nil {
176
177
178
179
180
		var pErr *fs.PathError
		if errors.As(err, &pErr) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
181
182
183
184
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

185
186
187
188
189
	workDir := c.GetString("workDir")

	// TODO: set this duration from the request if specified
	sessionDuration := defaultSessionDuration
	if err := load(c.Request.Context(), workDir, model, req.Options, sessionDuration); err != nil {
190
191
192
193
		if errors.Is(err, api.ErrInvalidOpts) {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
194
195
196
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
197

Michael Yang's avatar
Michael Yang committed
198
199
	checkpointLoaded := time.Now()

200
201
202
203
204
205
206
	prompt := req.Prompt
	if !req.Raw {
		prompt, err = model.Prompt(req)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
Michael Yang's avatar
Michael Yang committed
207
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
208

Michael Yang's avatar
Michael Yang committed
209
210
211
	ch := make(chan any)
	go func() {
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
212
213
214
215
216
217
		// an empty request loads the model
		if req.Prompt == "" && req.Template == "" && req.System == "" {
			ch <- api.GenerateResponse{CreatedAt: time.Now().UTC(), Model: req.Model, Done: true}
			return
		}

Michael Yang's avatar
Michael Yang committed
218
		fn := func(r api.GenerateResponse) {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
219
220
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)
Michael Yang's avatar
Michael Yang committed
221

Michael Yang's avatar
Michael Yang committed
222
223
224
			r.Model = req.Model
			r.CreatedAt = time.Now().UTC()
			if r.Done {
Michael Yang's avatar
Michael Yang committed
225
226
				r.TotalDuration = time.Since(checkpointStart)
				r.LoadDuration = checkpointLoaded.Sub(checkpointStart)
Michael Yang's avatar
Michael Yang committed
227
228
			}

229
230
231
232
233
			if req.Raw {
				// in raw mode the client must manage history on their own
				r.Context = nil
			}

Michael Yang's avatar
Michael Yang committed
234
			ch <- r
Michael Yang's avatar
Michael Yang committed
235
236
		}

237
		if err := loaded.runner.Predict(c.Request.Context(), req.Context, prompt, req.Format, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
238
			ch <- gin.H{"error": err.Error()}
Michael Yang's avatar
Michael Yang committed
239
		}
Michael Yang's avatar
Michael Yang committed
240
	}()
Michael Yang's avatar
Michael Yang committed
241

242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
	if req.Stream != nil && !*req.Stream {
		var response api.GenerateResponse
		generated := ""
		for resp := range ch {
			if r, ok := resp.(api.GenerateResponse); ok {
				generated += r.Response
				response = r
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}
		response.Response = generated
		c.JSON(http.StatusOK, response)
		return
	}

Michael Yang's avatar
Michael Yang committed
259
	streamResponse(c, ch)
Michael Yang's avatar
Michael Yang committed
260
}
Michael Yang's avatar
Michael Yang committed
261

Bruce MacDonald's avatar
Bruce MacDonald committed
262
263
264
265
266
func EmbeddingHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	var req api.EmbeddingRequest
Michael Yang's avatar
Michael Yang committed
267
268
269
270
271
272
273
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
274
275
276
		return
	}

277
278
279
280
281
	if req.Model == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
282
283
284
285
286
	model, err := GetModel(req.Model)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
287
288
289

	workDir := c.GetString("workDir")
	if err := load(c.Request.Context(), workDir, model, req.Options, 5*time.Minute); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
290
291
292
293
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

294
	if !loaded.Options.EmbeddingOnly {
Bruce MacDonald's avatar
Bruce MacDonald committed
295
296
297
298
		c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
		return
	}

299
	embedding, err := loaded.runner.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
300
301
302
303
304
305
306
307
308
309
310
311
	if err != nil {
		log.Printf("embedding generation failed: %v", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
		return
	}

	resp := api.EmbeddingResponse{
		Embedding: embedding,
	}
	c.JSON(http.StatusOK, resp)
}

312
func PullModelHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
313
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
314
315
316
317
318
319
320
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
321
322
323
		return
	}

324
325
326
327
328
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

329
330
331
	ch := make(chan any)
	go func() {
		defer close(ch)
332
333
		fn := func(r api.ProgressResponse) {
			ch <- r
334
		}
335

336
337
338
339
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

340
341
342
343
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

		if err := PullModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
344
			ch <- gin.H{"error": err.Error()}
345
346
347
		}
	}()

348
349
350
351
352
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

353
354
355
	streamResponse(c, ch)
}

356
func PushModelHandler(c *gin.Context) {
357
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
358
359
360
361
362
363
364
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
365
366
		return
	}
Michael Yang's avatar
Michael Yang committed
367

368
369
370
371
372
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

373
374
375
	ch := make(chan any)
	go func() {
		defer close(ch)
376
377
		fn := func(r api.ProgressResponse) {
			ch <- r
378
		}
379

380
381
382
383
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
384
385
386
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

387
		if err := PushModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
388
			ch <- gin.H{"error": err.Error()}
389
390
391
		}
	}()

392
393
394
395
396
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

397
398
399
	streamResponse(c, ch)
}

400
func CreateModelHandler(c *gin.Context) {
401
	var req api.CreateRequest
Michael Yang's avatar
Michael Yang committed
402
403
404
405
406
407
408
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
409
		return
410
411
	}

412
413
414
415
416
	if req.Name == "" || req.Path == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name and path are required"})
		return
	}

Michael Yang's avatar
Michael Yang committed
417
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
418
419
	go func() {
		defer close(ch)
420
421
		fn := func(resp api.ProgressResponse) {
			ch <- resp
422
423
		}

424
425
426
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

427
		if err := CreateModel(ctx, req.Name, req.Path, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
428
			ch <- gin.H{"error": err.Error()}
429
		}
Michael Yang's avatar
Michael Yang committed
430
	}()
Michael Yang's avatar
Michael Yang committed
431

432
433
434
435
436
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
437
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
438
439
}

440
441
func DeleteModelHandler(c *gin.Context) {
	var req api.DeleteRequest
Michael Yang's avatar
Michael Yang committed
442
443
444
445
446
447
448
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
449
450
451
		return
	}

452
453
454
455
456
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

457
458
459
460
	if err := DeleteModel(req.Name); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
461
462
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
463
464
		return
	}
Michael Yang's avatar
Michael Yang committed
465
466
467
468
469
470
471
472
473
474
475
476

	manifestsPath, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	if err := PruneDirectory(manifestsPath); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

477
	c.JSON(http.StatusOK, nil)
478
479
}

Patrick Devine's avatar
Patrick Devine committed
480
481
func ShowModelHandler(c *gin.Context) {
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
482
483
484
485
486
487
488
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
489
490
491
		return
	}

492
493
494
495
496
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
	resp, err := GetModelInfo(req.Name)
	if err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

func GetModelInfo(name string) (*api.ShowResponse, error) {
	model, err := GetModel(name)
	if err != nil {
		return nil, err
	}

	resp := &api.ShowResponse{
		License:  strings.Join(model.License, "\n"),
		System:   model.System,
		Template: model.Template,
	}

	mf, err := ShowModelfile(model)
	if err != nil {
		return nil, err
	}

	resp.Modelfile = mf

	var params []string
	cs := 30
	for k, v := range model.Options {
		switch val := v.(type) {
		case string:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
		case int:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
		case float64:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
		case bool:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
		case []interface{}:
			for _, nv := range val {
				switch nval := nv.(type) {
				case string:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
				case int:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
				case float64:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
				case bool:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
				}
			}
		}
	}
	resp.Parameters = strings.Join(params, "\n")

	return resp, nil
}

561
func ListModelsHandler(c *gin.Context) {
562
	models := make([]api.ModelResponse, 0)
Patrick Devine's avatar
Patrick Devine committed
563
564
565
566
567
	fp, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
568
569

	walkFunc := func(path string, info os.FileInfo, _ error) error {
Patrick Devine's avatar
Patrick Devine committed
570
		if !info.IsDir() {
Michael Yang's avatar
Michael Yang committed
571
572
573
			dir, file := filepath.Split(path)
			dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
			tag := strings.Join([]string{dir, file}, ":")
574

575
			mp := ParseModelPath(tag)
Patrick Devine's avatar
Patrick Devine committed
576
			manifest, digest, err := GetManifest(mp)
Patrick Devine's avatar
Patrick Devine committed
577
			if err != nil {
578
579
				log.Printf("skipping file: %s", fp)
				return nil
Patrick Devine's avatar
Patrick Devine committed
580
			}
Michael Yang's avatar
Michael Yang committed
581
582

			models = append(models, api.ModelResponse{
Patrick Devine's avatar
Patrick Devine committed
583
584
				Name:       mp.GetShortTagname(),
				Size:       manifest.GetTotalSize(),
Patrick Devine's avatar
Patrick Devine committed
585
				Digest:     digest,
Michael Yang's avatar
Michael Yang committed
586
587
				ModifiedAt: info.ModTime(),
			})
Patrick Devine's avatar
Patrick Devine committed
588
		}
Michael Yang's avatar
Michael Yang committed
589

Patrick Devine's avatar
Patrick Devine committed
590
		return nil
Michael Yang's avatar
Michael Yang committed
591
592
593
	}

	if err := filepath.Walk(fp, walkFunc); err != nil {
Patrick Devine's avatar
Patrick Devine committed
594
595
596
597
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
598
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
599
600
}

Patrick Devine's avatar
Patrick Devine committed
601
602
func CopyModelHandler(c *gin.Context) {
	var req api.CopyRequest
Michael Yang's avatar
Michael Yang committed
603
604
605
606
607
608
609
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
610
611
612
		return
	}

613
614
615
616
617
	if req.Source == "" || req.Destination == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "source add destination are required"})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
618
619
620
621
622
623
624
625
626
627
	if err := CopyModel(req.Source, req.Destination); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}
}

Michael Yang's avatar
Michael Yang committed
628
629
630
631
632
633
634
var defaultAllowOrigins = []string{
	"localhost",
	"127.0.0.1",
	"0.0.0.0",
}

func Serve(ln net.Listener, allowOrigins []string) error {
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
	if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		// clean up unused layers and manifests
		if err := PruneLayers(); err != nil {
			return err
		}

		manifestsPath, err := GetManifestPath()
		if err != nil {
			return err
		}

		if err := PruneDirectory(manifestsPath); err != nil {
			return err
		}
	}

Michael Yang's avatar
Michael Yang committed
651
652
	config := cors.DefaultConfig()
	config.AllowWildcard = true
Michael Yang's avatar
Michael Yang committed
653
654
655
656
657
658
659
660
661
662

	config.AllowOrigins = allowOrigins
	for _, allowOrigin := range defaultAllowOrigins {
		config.AllowOrigins = append(config.AllowOrigins,
			fmt.Sprintf("http://%s", allowOrigin),
			fmt.Sprintf("https://%s", allowOrigin),
			fmt.Sprintf("http://%s:*", allowOrigin),
			fmt.Sprintf("https://%s:*", allowOrigin),
		)
	}
Michael Yang's avatar
Michael Yang committed
663

664
665
666
667
668
669
	workDir, err := os.MkdirTemp("", "ollama")
	if err != nil {
		return err
	}
	defer os.RemoveAll(workDir)

Bruce MacDonald's avatar
Bruce MacDonald committed
670
	r := gin.Default()
671
672
673
674
675
676
677
	r.Use(
		cors.New(config),
		func(c *gin.Context) {
			c.Set("workDir", workDir)
			c.Next()
		},
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
678

679
680
	r.POST("/api/pull", PullModelHandler)
	r.POST("/api/generate", GenerateHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
681
	r.POST("/api/embeddings", EmbeddingHandler)
682
683
	r.POST("/api/create", CreateModelHandler)
	r.POST("/api/push", PushModelHandler)
Patrick Devine's avatar
Patrick Devine committed
684
	r.POST("/api/copy", CopyModelHandler)
685
	r.DELETE("/api/delete", DeleteModelHandler)
Patrick Devine's avatar
Patrick Devine committed
686
	r.POST("/api/show", ShowModelHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
687

Michael Yang's avatar
Michael Yang committed
688
689
690
691
692
693
694
695
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

		r.Handle(method, "/api/tags", ListModelsHandler)
	}

Michael Yang's avatar
Michael Yang committed
696
	log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
697
698
699
700
	s := &http.Server{
		Handler: r,
	}

701
702
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
703
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
704
705
	go func() {
		<-signals
706
707
		if loaded.runner != nil {
			loaded.runner.Close()
708
		}
709
		os.RemoveAll(workDir)
710
711
712
		os.Exit(0)
	}()

713
714
715
	if runtime.GOOS == "linux" {
		// check compatibility to log warnings
		if _, err := llm.CheckVRAM(); err != nil {
716
			log.Printf("Warning: GPU support may not be enabled, check you have installed GPU drivers: %v", err)
717
718
719
		}
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
720
721
	return s.Serve(ln)
}
Michael Yang's avatar
Michael Yang committed
722

723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
			if errorMsg, ok := r["error"].(string); ok {
				c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
				return
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in progress response"})
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
748
func streamResponse(c *gin.Context, ch chan any) {
749
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
750
751
752
753
754
755
756
757
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
758
			log.Printf("streamResponse: json.Marshal failed with %s", err)
Michael Yang's avatar
Michael Yang committed
759
760
761
			return false
		}

762
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
763
764
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
765
			log.Printf("streamResponse: w.Write failed with %s", err)
Michael Yang's avatar
Michael Yang committed
766
767
768
769
770
771
			return false
		}

		return true
	})
}