routes.go 17.8 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
4
	"context"
Michael Yang's avatar
Michael Yang committed
5
	"encoding/json"
6
	"errors"
7
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
8
	"io"
9
	"io/fs"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
10
11
12
	"log"
	"net"
	"net/http"
13
	"os"
14
	"os/signal"
Michael Yang's avatar
Michael Yang committed
15
	"path/filepath"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
16
	"reflect"
17
	"runtime"
Patrick Devine's avatar
Patrick Devine committed
18
	"strconv"
Michael Yang's avatar
Michael Yang committed
19
	"strings"
Michael Yang's avatar
Michael Yang committed
20
	"sync"
21
	"syscall"
22
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
23

Michael Yang's avatar
Michael Yang committed
24
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
25
26
	"github.com/gin-gonic/gin"

Jeffrey Morgan's avatar
Jeffrey Morgan committed
27
	"github.com/jmorganca/ollama/api"
28
	"github.com/jmorganca/ollama/llm"
Michael Yang's avatar
Michael Yang committed
29
	"github.com/jmorganca/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
30
31
)

Michael Yang's avatar
Michael Yang committed
32
33
34
35
36
37
38
39
40
41
42
43
44
45
var mode string = gin.DebugMode

func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
46
var loaded struct {
Michael Yang's avatar
Michael Yang committed
47
48
	mu sync.Mutex

49
	runner llm.LLM
Michael Yang's avatar
Michael Yang committed
50
51
52

	expireAt    time.Time
	expireTimer *time.Timer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
53

54
55
	*Model
	*api.Options
Michael Yang's avatar
Michael Yang committed
56
57
}

58
59
var defaultSessionDuration = 5 * time.Minute

Bruce MacDonald's avatar
Bruce MacDonald committed
60
// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
61
func load(ctx context.Context, workDir string, model *Model, reqOpts map[string]interface{}, sessionDuration time.Duration) error {
62
63
64
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		log.Printf("could not load model options: %v", err)
Bruce MacDonald's avatar
Bruce MacDonald committed
65
		return err
66
67
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
68
69
	if err := opts.FromMap(reqOpts); err != nil {
		return err
70
71
	}

72
	// check if the loaded model is still running in a subprocess, in case something unexpected happened
73
74
	if loaded.runner != nil {
		if err := loaded.runner.Ping(ctx); err != nil {
75
76
			log.Print("loaded llm process not responding, closing now")
			// the subprocess is no longer running, so close it
77
78
79
80
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
81
82
83
		}
	}

84
85
86
87
88
89
90
	needLoad := loaded.runner == nil || // is there a model loaded?
		loaded.ModelPath != model.ModelPath || // has the base model changed?
		!reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
		!reflect.DeepEqual(loaded.Options.Runner, opts.Runner) // have the runner options changed?

	if needLoad {
		if loaded.runner != nil {
91
			log.Println("changing loaded model")
92
93
94
95
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
96
		}
Michael Yang's avatar
Michael Yang committed
97

98
		llmRunner, err := llm.New(workDir, model.ModelPath, model.AdapterPaths, opts)
Michael Yang's avatar
Michael Yang committed
99
		if err != nil {
100
101
102
103
104
105
106
			// some older models are not compatible with newer versions of llama.cpp
			// show a generalized compatibility error until there is a better way to
			// check for model compatibility
			if strings.Contains(err.Error(), "failed to load model") {
				err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
			}

Bruce MacDonald's avatar
Bruce MacDonald committed
107
			return err
Michael Yang's avatar
Michael Yang committed
108
109
		}

110
111
112
		loaded.Model = model
		loaded.runner = llmRunner
		loaded.Options = &opts
Michael Yang's avatar
Michael Yang committed
113
	}
114

Michael Yang's avatar
Michael Yang committed
115
116
117
118
	// update options for the loaded llm
	// TODO(mxyng): this isn't thread safe, but it should be fine for now
	loaded.runner.SetOptions(opts)

Jeffrey Morgan's avatar
Jeffrey Morgan committed
119
	loaded.expireAt = time.Now().Add(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
120

Jeffrey Morgan's avatar
Jeffrey Morgan committed
121
122
123
124
	if loaded.expireTimer == nil {
		loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
			loaded.mu.Lock()
			defer loaded.mu.Unlock()
Michael Yang's avatar
Michael Yang committed
125

Jeffrey Morgan's avatar
Jeffrey Morgan committed
126
			if time.Now().Before(loaded.expireAt) {
Michael Yang's avatar
Michael Yang committed
127
128
129
				return
			}

130
131
			if loaded.runner != nil {
				loaded.runner.Close()
Michael Yang's avatar
Michael Yang committed
132
133
			}

134
135
136
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
137
		})
Michael Yang's avatar
Michael Yang committed
138
	}
139

Jeffrey Morgan's avatar
Jeffrey Morgan committed
140
	loaded.expireTimer.Reset(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
141
142
143
144
145
146
147
148
149
150
	return nil
}

func GenerateHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()

	var req api.GenerateRequest
Michael Yang's avatar
Michael Yang committed
151
152
153
154
155
156
157
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
158
159
160
		return
	}

161
162
163
164
165
	if req.Model == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
166
167
	model, err := GetModel(req.Model)
	if err != nil {
168
169
170
171
172
		var pErr *fs.PathError
		if errors.As(err, &pErr) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
173
174
175
176
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

177
178
179
180
181
	workDir := c.GetString("workDir")

	// TODO: set this duration from the request if specified
	sessionDuration := defaultSessionDuration
	if err := load(c.Request.Context(), workDir, model, req.Options, sessionDuration); err != nil {
182
183
184
185
		if errors.Is(err, api.ErrInvalidOpts) {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
186
187
188
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
189

Michael Yang's avatar
Michael Yang committed
190
191
	checkpointLoaded := time.Now()

192
	prompt, err := model.Prompt(req)
Michael Yang's avatar
Michael Yang committed
193
194
195
196
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
197

Michael Yang's avatar
Michael Yang committed
198
199
200
	ch := make(chan any)
	go func() {
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
201
202
203
204
205
206
		// an empty request loads the model
		if req.Prompt == "" && req.Template == "" && req.System == "" {
			ch <- api.GenerateResponse{CreatedAt: time.Now().UTC(), Model: req.Model, Done: true}
			return
		}

Michael Yang's avatar
Michael Yang committed
207
		fn := func(r api.GenerateResponse) {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
208
209
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)
Michael Yang's avatar
Michael Yang committed
210

Michael Yang's avatar
Michael Yang committed
211
212
213
			r.Model = req.Model
			r.CreatedAt = time.Now().UTC()
			if r.Done {
Michael Yang's avatar
Michael Yang committed
214
215
				r.TotalDuration = time.Since(checkpointStart)
				r.LoadDuration = checkpointLoaded.Sub(checkpointStart)
Michael Yang's avatar
Michael Yang committed
216
217
218
			}

			ch <- r
Michael Yang's avatar
Michael Yang committed
219
220
		}

Michael Yang's avatar
Michael Yang committed
221
222
		if err := loaded.runner.Predict(c.Request.Context(), req.Context, prompt, fn); err != nil {
			ch <- gin.H{"error": err.Error()}
Michael Yang's avatar
Michael Yang committed
223
		}
Michael Yang's avatar
Michael Yang committed
224
	}()
Michael Yang's avatar
Michael Yang committed
225

226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
	if req.Stream != nil && !*req.Stream {
		var response api.GenerateResponse
		generated := ""
		for resp := range ch {
			if r, ok := resp.(api.GenerateResponse); ok {
				generated += r.Response
				response = r
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}
		response.Response = generated
		c.JSON(http.StatusOK, response)
		return
	}

Michael Yang's avatar
Michael Yang committed
243
	streamResponse(c, ch)
Michael Yang's avatar
Michael Yang committed
244
}
Michael Yang's avatar
Michael Yang committed
245

Bruce MacDonald's avatar
Bruce MacDonald committed
246
247
248
249
250
func EmbeddingHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	var req api.EmbeddingRequest
Michael Yang's avatar
Michael Yang committed
251
252
253
254
255
256
257
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
258
259
260
		return
	}

261
262
263
264
265
	if req.Model == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
266
267
268
269
270
	model, err := GetModel(req.Model)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
271
272
273

	workDir := c.GetString("workDir")
	if err := load(c.Request.Context(), workDir, model, req.Options, 5*time.Minute); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
274
275
276
277
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

278
	if !loaded.Options.EmbeddingOnly {
Bruce MacDonald's avatar
Bruce MacDonald committed
279
280
281
282
		c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
		return
	}

283
	embedding, err := loaded.runner.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
284
285
286
287
288
289
290
291
292
293
294
295
	if err != nil {
		log.Printf("embedding generation failed: %v", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
		return
	}

	resp := api.EmbeddingResponse{
		Embedding: embedding,
	}
	c.JSON(http.StatusOK, resp)
}

296
func PullModelHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
297
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
298
299
300
301
302
303
304
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
305
306
307
		return
	}

308
309
310
311
312
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

313
314
315
	ch := make(chan any)
	go func() {
		defer close(ch)
316
317
		fn := func(r api.ProgressResponse) {
			ch <- r
318
		}
319

320
321
322
323
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

324
325
326
327
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

		if err := PullModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
328
			ch <- gin.H{"error": err.Error()}
329
330
331
		}
	}()

332
333
334
335
336
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

337
338
339
	streamResponse(c, ch)
}

340
func PushModelHandler(c *gin.Context) {
341
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
342
343
344
345
346
347
348
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
349
350
		return
	}
Michael Yang's avatar
Michael Yang committed
351

352
353
354
355
356
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

357
358
359
	ch := make(chan any)
	go func() {
		defer close(ch)
360
361
		fn := func(r api.ProgressResponse) {
			ch <- r
362
		}
363

364
365
366
367
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
368
369
370
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

371
		if err := PushModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
372
			ch <- gin.H{"error": err.Error()}
373
374
375
		}
	}()

376
377
378
379
380
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

381
382
383
	streamResponse(c, ch)
}

384
func CreateModelHandler(c *gin.Context) {
385
	var req api.CreateRequest
Michael Yang's avatar
Michael Yang committed
386
387
388
389
390
391
392
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
393
		return
394
395
	}

396
397
398
399
400
	if req.Name == "" || req.Path == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name and path are required"})
		return
	}

Michael Yang's avatar
Michael Yang committed
401
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
402
403
	go func() {
		defer close(ch)
404
405
		fn := func(resp api.ProgressResponse) {
			ch <- resp
406
407
		}

408
409
410
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

411
		if err := CreateModel(ctx, req.Name, req.Path, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
412
			ch <- gin.H{"error": err.Error()}
413
		}
Michael Yang's avatar
Michael Yang committed
414
	}()
Michael Yang's avatar
Michael Yang committed
415

416
417
418
419
420
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
421
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
422
423
}

424
425
func DeleteModelHandler(c *gin.Context) {
	var req api.DeleteRequest
Michael Yang's avatar
Michael Yang committed
426
427
428
429
430
431
432
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
433
434
435
		return
	}

436
437
438
439
440
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

441
442
443
444
	if err := DeleteModel(req.Name); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
445
446
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
447
448
		return
	}
Michael Yang's avatar
Michael Yang committed
449
450
451
452
453
454
455
456
457
458
459
460

	manifestsPath, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	if err := PruneDirectory(manifestsPath); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

461
	c.JSON(http.StatusOK, nil)
462
463
}

Patrick Devine's avatar
Patrick Devine committed
464
465
func ShowModelHandler(c *gin.Context) {
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
466
467
468
469
470
471
472
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
473
474
475
		return
	}

476
477
478
479
480
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
	resp, err := GetModelInfo(req.Name)
	if err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

func GetModelInfo(name string) (*api.ShowResponse, error) {
	model, err := GetModel(name)
	if err != nil {
		return nil, err
	}

	resp := &api.ShowResponse{
		License:  strings.Join(model.License, "\n"),
		System:   model.System,
		Template: model.Template,
	}

	mf, err := ShowModelfile(model)
	if err != nil {
		return nil, err
	}

	resp.Modelfile = mf

	var params []string
	cs := 30
	for k, v := range model.Options {
		switch val := v.(type) {
		case string:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
		case int:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
		case float64:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
		case bool:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
		case []interface{}:
			for _, nv := range val {
				switch nval := nv.(type) {
				case string:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
				case int:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
				case float64:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
				case bool:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
				}
			}
		}
	}
	resp.Parameters = strings.Join(params, "\n")

	return resp, nil
}

545
func ListModelsHandler(c *gin.Context) {
546
	models := make([]api.ModelResponse, 0)
Patrick Devine's avatar
Patrick Devine committed
547
548
549
550
551
	fp, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
552
553

	walkFunc := func(path string, info os.FileInfo, _ error) error {
Patrick Devine's avatar
Patrick Devine committed
554
		if !info.IsDir() {
Michael Yang's avatar
Michael Yang committed
555
556
557
			dir, file := filepath.Split(path)
			dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
			tag := strings.Join([]string{dir, file}, ":")
558

559
			mp := ParseModelPath(tag)
Patrick Devine's avatar
Patrick Devine committed
560
			manifest, digest, err := GetManifest(mp)
Patrick Devine's avatar
Patrick Devine committed
561
			if err != nil {
562
563
				log.Printf("skipping file: %s", fp)
				return nil
Patrick Devine's avatar
Patrick Devine committed
564
			}
Michael Yang's avatar
Michael Yang committed
565
566

			models = append(models, api.ModelResponse{
Patrick Devine's avatar
Patrick Devine committed
567
568
				Name:       mp.GetShortTagname(),
				Size:       manifest.GetTotalSize(),
Patrick Devine's avatar
Patrick Devine committed
569
				Digest:     digest,
Michael Yang's avatar
Michael Yang committed
570
571
				ModifiedAt: info.ModTime(),
			})
Patrick Devine's avatar
Patrick Devine committed
572
		}
Michael Yang's avatar
Michael Yang committed
573

Patrick Devine's avatar
Patrick Devine committed
574
		return nil
Michael Yang's avatar
Michael Yang committed
575
576
577
	}

	if err := filepath.Walk(fp, walkFunc); err != nil {
Patrick Devine's avatar
Patrick Devine committed
578
579
580
581
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
582
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
583
584
}

Patrick Devine's avatar
Patrick Devine committed
585
586
func CopyModelHandler(c *gin.Context) {
	var req api.CopyRequest
Michael Yang's avatar
Michael Yang committed
587
588
589
590
591
592
593
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
594
595
596
		return
	}

597
598
599
600
601
	if req.Source == "" || req.Destination == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "source add destination are required"})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
602
603
604
605
606
607
608
609
610
611
	if err := CopyModel(req.Source, req.Destination); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}
}

Michael Yang's avatar
Michael Yang committed
612
613
614
615
616
617
618
var defaultAllowOrigins = []string{
	"localhost",
	"127.0.0.1",
	"0.0.0.0",
}

func Serve(ln net.Listener, allowOrigins []string) error {
Michael Yang's avatar
Michael Yang committed
619
620
	config := cors.DefaultConfig()
	config.AllowWildcard = true
Michael Yang's avatar
Michael Yang committed
621
622
623
624
625
626
627
628
629
630

	config.AllowOrigins = allowOrigins
	for _, allowOrigin := range defaultAllowOrigins {
		config.AllowOrigins = append(config.AllowOrigins,
			fmt.Sprintf("http://%s", allowOrigin),
			fmt.Sprintf("https://%s", allowOrigin),
			fmt.Sprintf("http://%s:*", allowOrigin),
			fmt.Sprintf("https://%s:*", allowOrigin),
		)
	}
Michael Yang's avatar
Michael Yang committed
631

632
633
634
635
636
637
	workDir, err := os.MkdirTemp("", "ollama")
	if err != nil {
		return err
	}
	defer os.RemoveAll(workDir)

Bruce MacDonald's avatar
Bruce MacDonald committed
638
	r := gin.Default()
639
640
641
642
643
644
645
	r.Use(
		cors.New(config),
		func(c *gin.Context) {
			c.Set("workDir", workDir)
			c.Next()
		},
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
646

647
648
	r.POST("/api/pull", PullModelHandler)
	r.POST("/api/generate", GenerateHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
649
	r.POST("/api/embeddings", EmbeddingHandler)
650
651
	r.POST("/api/create", CreateModelHandler)
	r.POST("/api/push", PushModelHandler)
Patrick Devine's avatar
Patrick Devine committed
652
	r.POST("/api/copy", CopyModelHandler)
653
	r.DELETE("/api/delete", DeleteModelHandler)
Patrick Devine's avatar
Patrick Devine committed
654
	r.POST("/api/show", ShowModelHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
655

Michael Yang's avatar
Michael Yang committed
656
657
658
659
660
661
662
663
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

		r.Handle(method, "/api/tags", ListModelsHandler)
	}

Michael Yang's avatar
Michael Yang committed
664
	log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
665
666
667
668
	s := &http.Server{
		Handler: r,
	}

669
670
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
671
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
672
673
	go func() {
		<-signals
674
675
		if loaded.runner != nil {
			loaded.runner.Close()
676
		}
677
		os.RemoveAll(workDir)
678
679
680
		os.Exit(0)
	}()

681
682
683
	if runtime.GOOS == "linux" {
		// check compatibility to log warnings
		if _, err := llm.CheckVRAM(); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
684
			log.Printf("Warning: GPU support may not enabled, check you have installed install GPU drivers: %v", err)
685
686
687
		}
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
688
689
	return s.Serve(ln)
}
Michael Yang's avatar
Michael Yang committed
690

691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
			if errorMsg, ok := r["error"].(string); ok {
				c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
				return
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in progress response"})
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
716
func streamResponse(c *gin.Context, ch chan any) {
717
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
718
719
720
721
722
723
724
725
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
726
			log.Printf("streamResponse: json.Marshal failed with %s", err)
Michael Yang's avatar
Michael Yang committed
727
728
729
			return false
		}

730
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
731
732
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
733
			log.Printf("streamResponse: w.Write failed with %s", err)
Michael Yang's avatar
Michael Yang committed
734
735
736
737
738
739
			return false
		}

		return true
	})
}