routes.go 17.6 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
4
	"context"
Michael Yang's avatar
Michael Yang committed
5
	"encoding/json"
6
	"errors"
7
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
8
	"io"
9
	"io/fs"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
10
11
12
	"log"
	"net"
	"net/http"
13
	"os"
14
	"os/signal"
Michael Yang's avatar
Michael Yang committed
15
	"path/filepath"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
16
	"reflect"
17
	"runtime"
Patrick Devine's avatar
Patrick Devine committed
18
	"strconv"
Michael Yang's avatar
Michael Yang committed
19
	"strings"
Michael Yang's avatar
Michael Yang committed
20
	"sync"
21
	"syscall"
22
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
23

Michael Yang's avatar
Michael Yang committed
24
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
25
26
	"github.com/gin-gonic/gin"

Jeffrey Morgan's avatar
Jeffrey Morgan committed
27
	"github.com/jmorganca/ollama/api"
28
	"github.com/jmorganca/ollama/llm"
Michael Yang's avatar
Michael Yang committed
29
	"github.com/jmorganca/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
30
31
)

Michael Yang's avatar
Michael Yang committed
32
33
34
35
36
37
38
39
40
41
42
43
44
45
var mode string = gin.DebugMode

func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
46
var loaded struct {
Michael Yang's avatar
Michael Yang committed
47
48
	mu sync.Mutex

49
	runner llm.LLM
Michael Yang's avatar
Michael Yang committed
50
51
52

	expireAt    time.Time
	expireTimer *time.Timer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
53

54
55
	*Model
	*api.Options
Michael Yang's avatar
Michael Yang committed
56
57
}

58
59
var defaultSessionDuration = 5 * time.Minute

Bruce MacDonald's avatar
Bruce MacDonald committed
60
// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
61
func load(ctx context.Context, workDir string, model *Model, reqOpts map[string]interface{}, sessionDuration time.Duration) error {
62
63
64
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		log.Printf("could not load model options: %v", err)
Bruce MacDonald's avatar
Bruce MacDonald committed
65
		return err
66
67
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
68
69
	if err := opts.FromMap(reqOpts); err != nil {
		return err
70
71
	}

72
	// check if the loaded model is still running in a subprocess, in case something unexpected happened
73
74
	if loaded.runner != nil {
		if err := loaded.runner.Ping(ctx); err != nil {
75
76
			log.Print("loaded llm process not responding, closing now")
			// the subprocess is no longer running, so close it
77
78
79
80
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
81
82
83
		}
	}

84
85
86
87
88
89
90
	needLoad := loaded.runner == nil || // is there a model loaded?
		loaded.ModelPath != model.ModelPath || // has the base model changed?
		!reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
		!reflect.DeepEqual(loaded.Options.Runner, opts.Runner) // have the runner options changed?

	if needLoad {
		if loaded.runner != nil {
91
			log.Println("changing loaded model")
92
93
94
95
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
96
		}
Michael Yang's avatar
Michael Yang committed
97

98
		llmRunner, err := llm.New(workDir, model.ModelPath, model.AdapterPaths, opts)
Michael Yang's avatar
Michael Yang committed
99
		if err != nil {
100
101
102
103
104
105
106
			// some older models are not compatible with newer versions of llama.cpp
			// show a generalized compatibility error until there is a better way to
			// check for model compatibility
			if strings.Contains(err.Error(), "failed to load model") {
				err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
			}

Bruce MacDonald's avatar
Bruce MacDonald committed
107
			return err
Michael Yang's avatar
Michael Yang committed
108
109
		}

110
111
112
		loaded.Model = model
		loaded.runner = llmRunner
		loaded.Options = &opts
Michael Yang's avatar
Michael Yang committed
113
	}
114

Jeffrey Morgan's avatar
Jeffrey Morgan committed
115
	loaded.expireAt = time.Now().Add(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
116

Jeffrey Morgan's avatar
Jeffrey Morgan committed
117
118
119
120
	if loaded.expireTimer == nil {
		loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
			loaded.mu.Lock()
			defer loaded.mu.Unlock()
Michael Yang's avatar
Michael Yang committed
121

Jeffrey Morgan's avatar
Jeffrey Morgan committed
122
			if time.Now().Before(loaded.expireAt) {
Michael Yang's avatar
Michael Yang committed
123
124
125
				return
			}

126
127
			if loaded.runner != nil {
				loaded.runner.Close()
Michael Yang's avatar
Michael Yang committed
128
129
			}

130
131
132
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
133
		})
Michael Yang's avatar
Michael Yang committed
134
	}
135

Jeffrey Morgan's avatar
Jeffrey Morgan committed
136
	loaded.expireTimer.Reset(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
137
138
139
140
141
142
143
144
145
146
	return nil
}

func GenerateHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()

	var req api.GenerateRequest
Michael Yang's avatar
Michael Yang committed
147
148
149
150
151
152
153
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
154
155
156
		return
	}

157
158
159
160
161
	if req.Model == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
162
163
	model, err := GetModel(req.Model)
	if err != nil {
164
165
166
167
168
		var pErr *fs.PathError
		if errors.As(err, &pErr) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
169
170
171
172
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

173
174
175
176
177
	workDir := c.GetString("workDir")

	// TODO: set this duration from the request if specified
	sessionDuration := defaultSessionDuration
	if err := load(c.Request.Context(), workDir, model, req.Options, sessionDuration); err != nil {
178
179
180
181
		if errors.Is(err, api.ErrInvalidOpts) {
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
182
183
184
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
185

Michael Yang's avatar
Michael Yang committed
186
187
	checkpointLoaded := time.Now()

188
	prompt, err := model.Prompt(req)
Michael Yang's avatar
Michael Yang committed
189
190
191
192
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
193

Michael Yang's avatar
Michael Yang committed
194
195
196
	ch := make(chan any)
	go func() {
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
197
198
199
200
201
202
		// an empty request loads the model
		if req.Prompt == "" && req.Template == "" && req.System == "" {
			ch <- api.GenerateResponse{CreatedAt: time.Now().UTC(), Model: req.Model, Done: true}
			return
		}

Michael Yang's avatar
Michael Yang committed
203
		fn := func(r api.GenerateResponse) {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
204
205
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)
Michael Yang's avatar
Michael Yang committed
206

Michael Yang's avatar
Michael Yang committed
207
208
209
			r.Model = req.Model
			r.CreatedAt = time.Now().UTC()
			if r.Done {
Michael Yang's avatar
Michael Yang committed
210
211
				r.TotalDuration = time.Since(checkpointStart)
				r.LoadDuration = checkpointLoaded.Sub(checkpointStart)
Michael Yang's avatar
Michael Yang committed
212
213
214
			}

			ch <- r
Michael Yang's avatar
Michael Yang committed
215
216
		}

Michael Yang's avatar
Michael Yang committed
217
218
		if err := loaded.runner.Predict(c.Request.Context(), req.Context, prompt, fn); err != nil {
			ch <- gin.H{"error": err.Error()}
Michael Yang's avatar
Michael Yang committed
219
		}
Michael Yang's avatar
Michael Yang committed
220
	}()
Michael Yang's avatar
Michael Yang committed
221

222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
	if req.Stream != nil && !*req.Stream {
		var response api.GenerateResponse
		generated := ""
		for resp := range ch {
			if r, ok := resp.(api.GenerateResponse); ok {
				generated += r.Response
				response = r
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}
		response.Response = generated
		c.JSON(http.StatusOK, response)
		return
	}

Michael Yang's avatar
Michael Yang committed
239
	streamResponse(c, ch)
Michael Yang's avatar
Michael Yang committed
240
}
Michael Yang's avatar
Michael Yang committed
241

Bruce MacDonald's avatar
Bruce MacDonald committed
242
243
244
245
246
func EmbeddingHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	var req api.EmbeddingRequest
Michael Yang's avatar
Michael Yang committed
247
248
249
250
251
252
253
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
254
255
256
		return
	}

257
258
259
260
261
	if req.Model == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
262
263
264
265
266
	model, err := GetModel(req.Model)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
267
268
269

	workDir := c.GetString("workDir")
	if err := load(c.Request.Context(), workDir, model, req.Options, 5*time.Minute); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
270
271
272
273
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

274
	if !loaded.Options.EmbeddingOnly {
Bruce MacDonald's avatar
Bruce MacDonald committed
275
276
277
278
		c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
		return
	}

279
	embedding, err := loaded.runner.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
280
281
282
283
284
285
286
287
288
289
290
291
	if err != nil {
		log.Printf("embedding generation failed: %v", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
		return
	}

	resp := api.EmbeddingResponse{
		Embedding: embedding,
	}
	c.JSON(http.StatusOK, resp)
}

292
func PullModelHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
293
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
294
295
296
297
298
299
300
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
301
302
303
		return
	}

304
305
306
307
308
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

309
310
311
	ch := make(chan any)
	go func() {
		defer close(ch)
312
313
		fn := func(r api.ProgressResponse) {
			ch <- r
314
		}
315

316
317
318
319
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

320
321
322
323
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

		if err := PullModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
324
			ch <- gin.H{"error": err.Error()}
325
326
327
		}
	}()

328
329
330
331
332
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

333
334
335
	streamResponse(c, ch)
}

336
func PushModelHandler(c *gin.Context) {
337
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
338
339
340
341
342
343
344
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
345
346
		return
	}
Michael Yang's avatar
Michael Yang committed
347

348
349
350
351
352
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

353
354
355
	ch := make(chan any)
	go func() {
		defer close(ch)
356
357
		fn := func(r api.ProgressResponse) {
			ch <- r
358
		}
359

360
361
362
363
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

364
365
		ctx := context.Background()
		if err := PushModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
366
			ch <- gin.H{"error": err.Error()}
367
368
369
		}
	}()

370
371
372
373
374
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

375
376
377
	streamResponse(c, ch)
}

378
func CreateModelHandler(c *gin.Context) {
379
	var req api.CreateRequest
Michael Yang's avatar
Michael Yang committed
380
381
382
383
384
385
386
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
387
		return
388
389
	}

390
391
392
393
394
	if req.Name == "" || req.Path == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name and path are required"})
		return
	}

Michael Yang's avatar
Michael Yang committed
395
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
396
397
	go func() {
		defer close(ch)
398
399
		fn := func(resp api.ProgressResponse) {
			ch <- resp
400
401
		}

402
403
404
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

405
		if err := CreateModel(ctx, req.Name, req.Path, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
406
			ch <- gin.H{"error": err.Error()}
407
		}
Michael Yang's avatar
Michael Yang committed
408
	}()
Michael Yang's avatar
Michael Yang committed
409

410
411
412
413
414
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
415
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
416
417
}

418
419
func DeleteModelHandler(c *gin.Context) {
	var req api.DeleteRequest
Michael Yang's avatar
Michael Yang committed
420
421
422
423
424
425
426
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
427
428
429
		return
	}

430
431
432
433
434
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

435
436
437
438
	if err := DeleteModel(req.Name); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
439
440
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
441
442
		return
	}
Michael Yang's avatar
Michael Yang committed
443
444
445
446
447
448
449
450
451
452
453
454

	manifestsPath, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	if err := PruneDirectory(manifestsPath); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

455
	c.JSON(http.StatusOK, nil)
456
457
}

Patrick Devine's avatar
Patrick Devine committed
458
459
func ShowModelHandler(c *gin.Context) {
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
460
461
462
463
464
465
466
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
467
468
469
		return
	}

470
471
472
473
474
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
	resp, err := GetModelInfo(req.Name)
	if err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

func GetModelInfo(name string) (*api.ShowResponse, error) {
	model, err := GetModel(name)
	if err != nil {
		return nil, err
	}

	resp := &api.ShowResponse{
		License:  strings.Join(model.License, "\n"),
		System:   model.System,
		Template: model.Template,
	}

	mf, err := ShowModelfile(model)
	if err != nil {
		return nil, err
	}

	resp.Modelfile = mf

	var params []string
	cs := 30
	for k, v := range model.Options {
		switch val := v.(type) {
		case string:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
		case int:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
		case float64:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
		case bool:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
		case []interface{}:
			for _, nv := range val {
				switch nval := nv.(type) {
				case string:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
				case int:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
				case float64:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
				case bool:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
				}
			}
		}
	}
	resp.Parameters = strings.Join(params, "\n")

	return resp, nil
}

539
func ListModelsHandler(c *gin.Context) {
540
	models := make([]api.ModelResponse, 0)
Patrick Devine's avatar
Patrick Devine committed
541
542
543
544
545
	fp, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
546
547

	walkFunc := func(path string, info os.FileInfo, _ error) error {
Patrick Devine's avatar
Patrick Devine committed
548
		if !info.IsDir() {
Michael Yang's avatar
Michael Yang committed
549
550
551
			dir, file := filepath.Split(path)
			dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
			tag := strings.Join([]string{dir, file}, ":")
552

553
			mp := ParseModelPath(tag)
Patrick Devine's avatar
Patrick Devine committed
554
			manifest, digest, err := GetManifest(mp)
Patrick Devine's avatar
Patrick Devine committed
555
			if err != nil {
556
557
				log.Printf("skipping file: %s", fp)
				return nil
Patrick Devine's avatar
Patrick Devine committed
558
			}
Michael Yang's avatar
Michael Yang committed
559
560

			models = append(models, api.ModelResponse{
Patrick Devine's avatar
Patrick Devine committed
561
562
				Name:       mp.GetShortTagname(),
				Size:       manifest.GetTotalSize(),
Patrick Devine's avatar
Patrick Devine committed
563
				Digest:     digest,
Michael Yang's avatar
Michael Yang committed
564
565
				ModifiedAt: info.ModTime(),
			})
Patrick Devine's avatar
Patrick Devine committed
566
		}
Michael Yang's avatar
Michael Yang committed
567

Patrick Devine's avatar
Patrick Devine committed
568
		return nil
Michael Yang's avatar
Michael Yang committed
569
570
571
	}

	if err := filepath.Walk(fp, walkFunc); err != nil {
Patrick Devine's avatar
Patrick Devine committed
572
573
574
575
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
576
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
577
578
}

Patrick Devine's avatar
Patrick Devine committed
579
580
func CopyModelHandler(c *gin.Context) {
	var req api.CopyRequest
Michael Yang's avatar
Michael Yang committed
581
582
583
584
585
586
587
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
588
589
590
		return
	}

591
592
593
594
595
	if req.Source == "" || req.Destination == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "source add destination are required"})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
596
597
598
599
600
601
602
603
604
605
	if err := CopyModel(req.Source, req.Destination); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}
}

Michael Yang's avatar
Michael Yang committed
606
607
608
609
610
611
612
var defaultAllowOrigins = []string{
	"localhost",
	"127.0.0.1",
	"0.0.0.0",
}

func Serve(ln net.Listener, allowOrigins []string) error {
Michael Yang's avatar
Michael Yang committed
613
614
	config := cors.DefaultConfig()
	config.AllowWildcard = true
Michael Yang's avatar
Michael Yang committed
615
616
617
618
619
620
621
622
623
624

	config.AllowOrigins = allowOrigins
	for _, allowOrigin := range defaultAllowOrigins {
		config.AllowOrigins = append(config.AllowOrigins,
			fmt.Sprintf("http://%s", allowOrigin),
			fmt.Sprintf("https://%s", allowOrigin),
			fmt.Sprintf("http://%s:*", allowOrigin),
			fmt.Sprintf("https://%s:*", allowOrigin),
		)
	}
Michael Yang's avatar
Michael Yang committed
625

626
627
628
629
630
631
	workDir, err := os.MkdirTemp("", "ollama")
	if err != nil {
		return err
	}
	defer os.RemoveAll(workDir)

Bruce MacDonald's avatar
Bruce MacDonald committed
632
	r := gin.Default()
633
634
635
636
637
638
639
	r.Use(
		cors.New(config),
		func(c *gin.Context) {
			c.Set("workDir", workDir)
			c.Next()
		},
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
640

641
642
	r.POST("/api/pull", PullModelHandler)
	r.POST("/api/generate", GenerateHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
643
	r.POST("/api/embeddings", EmbeddingHandler)
644
645
	r.POST("/api/create", CreateModelHandler)
	r.POST("/api/push", PushModelHandler)
Patrick Devine's avatar
Patrick Devine committed
646
	r.POST("/api/copy", CopyModelHandler)
647
	r.DELETE("/api/delete", DeleteModelHandler)
Patrick Devine's avatar
Patrick Devine committed
648
	r.POST("/api/show", ShowModelHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
649

Michael Yang's avatar
Michael Yang committed
650
651
652
653
654
655
656
657
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

		r.Handle(method, "/api/tags", ListModelsHandler)
	}

Michael Yang's avatar
Michael Yang committed
658
	log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
659
660
661
662
	s := &http.Server{
		Handler: r,
	}

663
664
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
665
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
666
667
	go func() {
		<-signals
668
669
		if loaded.runner != nil {
			loaded.runner.Close()
670
		}
671
		os.RemoveAll(workDir)
672
673
674
		os.Exit(0)
	}()

675
676
677
	if runtime.GOOS == "linux" {
		// check compatibility to log warnings
		if _, err := llm.CheckVRAM(); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
678
			log.Printf("Warning: GPU support may not enabled, check you have installed install GPU drivers: %v", err)
679
680
681
		}
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
682
683
	return s.Serve(ln)
}
Michael Yang's avatar
Michael Yang committed
684

685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
			if errorMsg, ok := r["error"].(string); ok {
				c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
				return
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in progress response"})
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
710
func streamResponse(c *gin.Context, ch chan any) {
711
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
712
713
714
715
716
717
718
719
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
720
			log.Printf("streamResponse: json.Marshal failed with %s", err)
Michael Yang's avatar
Michael Yang committed
721
722
723
			return false
		}

724
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
725
726
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
727
			log.Printf("streamResponse: w.Write failed with %s", err)
Michael Yang's avatar
Michael Yang committed
728
729
730
731
732
733
			return false
		}

		return true
	})
}