routes.go 28.3 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
4
	"context"
Michael Yang's avatar
Michael Yang committed
5
	"encoding/json"
6
	"errors"
7
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
8
	"io"
9
	"io/fs"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
10
11
12
	"log"
	"net"
	"net/http"
13
	"os"
14
	"os/signal"
Michael Yang's avatar
Michael Yang committed
15
	"path/filepath"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
16
	"reflect"
17
	"runtime"
Patrick Devine's avatar
Patrick Devine committed
18
	"strconv"
Michael Yang's avatar
Michael Yang committed
19
	"strings"
Michael Yang's avatar
Michael Yang committed
20
	"sync"
21
	"syscall"
22
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
23

Michael Yang's avatar
Michael Yang committed
24
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
25
26
	"github.com/gin-gonic/gin"

Jeffrey Morgan's avatar
Jeffrey Morgan committed
27
	"github.com/jmorganca/ollama/api"
28
	"github.com/jmorganca/ollama/gpu"
29
	"github.com/jmorganca/ollama/llm"
Michael Yang's avatar
Michael Yang committed
30
	"github.com/jmorganca/ollama/parser"
Michael Yang's avatar
Michael Yang committed
31
	"github.com/jmorganca/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
32
33
)

Michael Yang's avatar
Michael Yang committed
34
35
var mode string = gin.DebugMode

36
37
38
39
type Server struct {
	WorkDir string
}

Michael Yang's avatar
Michael Yang committed
40
41
42
43
44
45
46
47
48
49
50
51
func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
52
var loaded struct {
Michael Yang's avatar
Michael Yang committed
53
54
	mu sync.Mutex

55
	runner llm.LLM
Michael Yang's avatar
Michael Yang committed
56
57
58

	expireAt    time.Time
	expireTimer *time.Timer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
59

60
61
	*Model
	*api.Options
Michael Yang's avatar
Michael Yang committed
62
63
}

64
65
var defaultSessionDuration = 5 * time.Minute

Bruce MacDonald's avatar
Bruce MacDonald committed
66
// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
67
func load(c *gin.Context, model *Model, opts api.Options, sessionDuration time.Duration) error {
Bruce MacDonald's avatar
Bruce MacDonald committed
68
69
	workDir := c.GetString("workDir")

70
71
72
73
74
75
76
	needLoad := loaded.runner == nil || // is there a model loaded?
		loaded.ModelPath != model.ModelPath || // has the base model changed?
		!reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
		!reflect.DeepEqual(loaded.Options.Runner, opts.Runner) // have the runner options changed?

	if needLoad {
		if loaded.runner != nil {
77
			log.Println("changing loaded model")
78
79
80
81
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
82
		}
Michael Yang's avatar
Michael Yang committed
83

Michael Yang's avatar
Michael Yang committed
84
		llmRunner, err := llm.New(workDir, model.ModelPath, model.AdapterPaths, model.ProjectorPaths, opts)
Michael Yang's avatar
Michael Yang committed
85
		if err != nil {
86
87
88
			// some older models are not compatible with newer versions of llama.cpp
			// show a generalized compatibility error until there is a better way to
			// check for model compatibility
Bruce MacDonald's avatar
Bruce MacDonald committed
89
			if errors.Is(llm.ErrUnsupportedFormat, err) || strings.Contains(err.Error(), "failed to load model") {
90
91
92
				err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
			}

93
			return err
Michael Yang's avatar
Michael Yang committed
94
95
		}

96
97
98
		loaded.Model = model
		loaded.runner = llmRunner
		loaded.Options = &opts
Michael Yang's avatar
Michael Yang committed
99
	}
100

Jeffrey Morgan's avatar
Jeffrey Morgan committed
101
	loaded.expireAt = time.Now().Add(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
102

Jeffrey Morgan's avatar
Jeffrey Morgan committed
103
104
105
106
	if loaded.expireTimer == nil {
		loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
			loaded.mu.Lock()
			defer loaded.mu.Unlock()
Michael Yang's avatar
Michael Yang committed
107

Jeffrey Morgan's avatar
Jeffrey Morgan committed
108
			if time.Now().Before(loaded.expireAt) {
Michael Yang's avatar
Michael Yang committed
109
110
111
				return
			}

112
113
			if loaded.runner != nil {
				loaded.runner.Close()
Michael Yang's avatar
Michael Yang committed
114
115
			}

116
117
118
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
119
		})
Michael Yang's avatar
Michael Yang committed
120
	}
121

Jeffrey Morgan's avatar
Jeffrey Morgan committed
122
	loaded.expireTimer.Reset(sessionDuration)
123
124
125
126
127
128
129
130
131
132
133
134
135
136
	return nil
}

func modelOptions(model *Model, requestOpts map[string]interface{}) (api.Options, error) {
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		return api.Options{}, err
	}

	if err := opts.FromMap(requestOpts); err != nil {
		return api.Options{}, err
	}

	return opts, nil
Bruce MacDonald's avatar
Bruce MacDonald committed
137
138
139
140
141
142
143
144
}

func GenerateHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()
	var req api.GenerateRequest
Michael Yang's avatar
Michael Yang committed
145
	err := c.ShouldBindJSON(&req)
Patrick Devine's avatar
Patrick Devine committed
146

Michael Yang's avatar
Michael Yang committed
147
148
149
150
151
152
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
153
154
155
		return
	}

156
157
158
	// validate the request
	switch {
	case req.Model == "":
159
160
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
161
162
163
	case len(req.Format) > 0 && req.Format != "json":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
		return
164
165
166
	case req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
		return
167
168
	}

169
	model, err := GetModel(req.Model)
Bruce MacDonald's avatar
Bruce MacDonald committed
170
	if err != nil {
171
		var pErr *fs.PathError
172
		if errors.As(err, &pErr) {
173
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
174
175
176
177
178
179
180
181
182
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	opts, err := modelOptions(model, req.Options)
	if err != nil {
		if errors.Is(err, api.ErrInvalidOpts) {
Bruce MacDonald's avatar
Bruce MacDonald committed
183
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
184
			return
185
		}
186
187
188
189
190
191
192
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	sessionDuration := defaultSessionDuration
	if err := load(c, model, opts, sessionDuration); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
193
194
195
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
196
197
	// an empty request loads the model
	if req.Prompt == "" && req.Template == "" && req.System == "" {
198
		c.JSON(http.StatusOK, api.GenerateResponse{
199
200
201
			CreatedAt: time.Now().UTC(),
			Model:     req.Model,
			Done:      true})
Bruce MacDonald's avatar
Bruce MacDonald committed
202
203
204
205
206
		return
	}

	checkpointLoaded := time.Now()

Bruce MacDonald's avatar
Bruce MacDonald committed
207
	var prompt string
208
	var promptVars PromptVars
Bruce MacDonald's avatar
Bruce MacDonald committed
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
	switch {
	case req.Raw:
		prompt = req.Prompt
	case req.Prompt != "":
		if req.Template != "" {
			// override the default model template
			model.Template = req.Template
		}

		var rebuild strings.Builder
		if req.Context != nil {
			// TODO: context is deprecated, at some point the context logic within this conditional should be removed
			prevCtx, err := loaded.runner.Decode(c.Request.Context(), req.Context)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}

			// Remove leading spaces from prevCtx if present
			prevCtx = strings.TrimPrefix(prevCtx, " ")
			rebuild.WriteString(prevCtx)
		}
231
		promptVars = PromptVars{
Bruce MacDonald's avatar
Bruce MacDonald committed
232
233
234
			System: req.System,
			Prompt: req.Prompt,
			First:  len(req.Context) == 0,
235
236
		}
		p, err := model.PreResponsePrompt(promptVars)
237
238
239
240
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
241
242
		rebuild.WriteString(p)
		prompt = rebuild.String()
Bruce MacDonald's avatar
Bruce MacDonald committed
243
244
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
245
	ch := make(chan any)
Bruce MacDonald's avatar
Bruce MacDonald committed
246
	var generated strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
247
248
249
	go func() {
		defer close(ch)

Bruce MacDonald's avatar
Bruce MacDonald committed
250
251
		fn := func(r llm.PredictResult) {
			// Update model expiration
Bruce MacDonald's avatar
Bruce MacDonald committed
252
253
254
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)

Bruce MacDonald's avatar
Bruce MacDonald committed
255
256
257
258
			// Build up the full response
			if _, err := generated.WriteString(r.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
259
260
			}

Bruce MacDonald's avatar
Bruce MacDonald committed
261
			resp := api.GenerateResponse{
262
				Model:     req.Model,
263
				CreatedAt: time.Now().UTC(),
264
265
				Done:      r.Done,
				Response:  r.Content,
Bruce MacDonald's avatar
Bruce MacDonald committed
266
267
268
269
270
271
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
Bruce MacDonald's avatar
Bruce MacDonald committed
272
273
			}

274
275
276
277
278
			if r.Done {
				resp.TotalDuration = time.Since(checkpointStart)
				resp.LoadDuration = checkpointLoaded.Sub(checkpointStart)

				if !req.Raw {
279
280
281
282
283
284
285
286
					// append the generated text to the history and template it if needed
					promptVars.Response = generated.String()
					result, err := model.PostResponseTemplate(promptVars)
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
					embd, err := loaded.runner.Encode(c.Request.Context(), prompt+result)
287
288
289
290
291
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
					resp.Context = embd
Bruce MacDonald's avatar
Bruce MacDonald committed
292
293
294
295
				}
			}

			ch <- resp
Bruce MacDonald's avatar
Bruce MacDonald committed
296
297
		}

Bruce MacDonald's avatar
Bruce MacDonald committed
298
299
		// Start prediction
		predictReq := llm.PredictOpts{
300
301
302
303
			Prompt:  prompt,
			Format:  req.Format,
			Images:  req.Images,
			Options: opts,
Bruce MacDonald's avatar
Bruce MacDonald committed
304
305
		}
		if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
306
307
308
309
310
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
311
312
		// Accumulate responses into the final response
		var final api.GenerateResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
313
		var sb strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
314
		for resp := range ch {
315
316
317
318
319
320
321
322
323
324
325
326
327
328
			switch r := resp.(type) {
			case api.GenerateResponse:
				sb.WriteString(r.Response)
				final = r
			case gin.H:
				if errorMsg, ok := r["error"].(string); ok {
					c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
					return
				} else {
					c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
					return
				}
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
Bruce MacDonald's avatar
Bruce MacDonald committed
329
330
331
				return
			}
		}
332
333
334

		final.Response = sb.String()
		c.JSON(http.StatusOK, final)
Bruce MacDonald's avatar
Bruce MacDonald committed
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
		return
	}

	streamResponse(c, ch)
}

func EmbeddingHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	var req api.EmbeddingRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if req.Model == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

361
	model, err := GetModel(req.Model)
Bruce MacDonald's avatar
Bruce MacDonald committed
362
	if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
363
		var pErr *fs.PathError
364
		if errors.As(err, &pErr) {
Bruce MacDonald's avatar
Bruce MacDonald committed
365
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
366
367
368
369
370
371
372
373
374
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	opts, err := modelOptions(model, req.Options)
	if err != nil {
		if errors.Is(err, api.ErrInvalidOpts) {
Bruce MacDonald's avatar
Bruce MacDonald committed
375
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
376
			return
Bruce MacDonald's avatar
Bruce MacDonald committed
377
		}
378
379
380
381
382
383
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	sessionDuration := defaultSessionDuration
	if err := load(c, model, opts, sessionDuration); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
384
385
386
		return
	}

387
	if !loaded.Options.EmbeddingOnly {
Bruce MacDonald's avatar
Bruce MacDonald committed
388
389
390
391
		c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
		return
	}

392
	embedding, err := loaded.runner.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
393
394
395
396
397
398
399
400
401
402
403
404
	if err != nil {
		log.Printf("embedding generation failed: %v", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
		return
	}

	resp := api.EmbeddingResponse{
		Embedding: embedding,
	}
	c.JSON(http.StatusOK, resp)
}

405
func PullModelHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
406
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
407
408
409
410
411
412
413
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
414
415
416
		return
	}

417
418
419
420
421
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

422
423
424
	ch := make(chan any)
	go func() {
		defer close(ch)
425
426
		fn := func(r api.ProgressResponse) {
			ch <- r
427
		}
428

429
430
431
432
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

433
434
435
436
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

		if err := PullModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
437
			ch <- gin.H{"error": err.Error()}
438
439
440
		}
	}()

441
442
443
444
445
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

446
447
448
	streamResponse(c, ch)
}

449
func PushModelHandler(c *gin.Context) {
450
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
451
452
453
454
455
456
457
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
458
459
		return
	}
Michael Yang's avatar
Michael Yang committed
460

461
462
463
464
465
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

466
467
468
	ch := make(chan any)
	go func() {
		defer close(ch)
469
470
		fn := func(r api.ProgressResponse) {
			ch <- r
471
		}
472

473
474
475
476
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
477
478
479
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

480
		if err := PushModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
481
			ch <- gin.H{"error": err.Error()}
482
483
484
		}
	}()

485
486
487
488
489
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

490
491
492
	streamResponse(c, ch)
}

493
func CreateModelHandler(c *gin.Context) {
494
	var req api.CreateRequest
Michael Yang's avatar
Michael Yang committed
495
496
497
498
499
500
501
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
502
		return
503
504
	}

Michael Yang's avatar
Michael Yang committed
505
506
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
507
508
509
		return
	}

510
511
	if err := ParseModelPath(req.Name).Validate(); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
512
513
514
		return
	}

Michael Yang's avatar
Michael Yang committed
515
516
	if req.Path == "" && req.Modelfile == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"})
Michael Yang's avatar
Michael Yang committed
517
518
		return
	}
Michael Yang's avatar
Michael Yang committed
519
520
521

	var modelfile io.Reader = strings.NewReader(req.Modelfile)
	if req.Path != "" && req.Modelfile == "" {
522
		mf, err := os.Open(req.Path)
Michael Yang's avatar
Michael Yang committed
523
524
525
526
		if err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)})
			return
		}
527
		defer mf.Close()
Michael Yang's avatar
Michael Yang committed
528

529
		modelfile = mf
Michael Yang's avatar
Michael Yang committed
530
	}
Michael Yang's avatar
Michael Yang committed
531
532
533
534
535
536
537

	commands, err := parser.Parse(modelfile)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
538
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
539
540
	go func() {
		defer close(ch)
541
542
		fn := func(resp api.ProgressResponse) {
			ch <- resp
543
544
		}

545
546
547
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

548
		if err := CreateModel(ctx, req.Name, filepath.Dir(req.Path), commands, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
549
			ch <- gin.H{"error": err.Error()}
550
		}
Michael Yang's avatar
Michael Yang committed
551
	}()
Michael Yang's avatar
Michael Yang committed
552

553
554
555
556
557
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
558
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
559
560
}

561
562
func DeleteModelHandler(c *gin.Context) {
	var req api.DeleteRequest
Michael Yang's avatar
Michael Yang committed
563
564
565
566
567
568
569
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
570
571
572
		return
	}

573
574
575
576
577
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

578
579
580
581
	if err := DeleteModel(req.Name); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
582
583
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
584
585
		return
	}
Michael Yang's avatar
Michael Yang committed
586
587
588
589
590
591
592
593
594
595
596
597

	manifestsPath, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	if err := PruneDirectory(manifestsPath); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

598
	c.JSON(http.StatusOK, nil)
599
600
}

Patrick Devine's avatar
Patrick Devine committed
601
602
func ShowModelHandler(c *gin.Context) {
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
603
604
605
606
607
608
609
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
610
611
612
		return
	}

613
614
615
	switch {
	case req.Model == "" && req.Name == "":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
616
		return
617
618
619
620
621
	case req.Model != "" && req.Name != "":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "both model and name are set"})
		return
	case req.Model == "" && req.Name != "":
		req.Model = req.Name
622
623
	}

624
	resp, err := GetModelInfo(req)
Patrick Devine's avatar
Patrick Devine committed
625
626
627
628
629
630
631
632
633
634
635
636
	if err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

637
638
func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
	model, err := GetModel(req.Model)
Patrick Devine's avatar
Patrick Devine committed
639
640
641
642
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
643
644
645
646
647
648
649
650
	modelDetails := api.ModelDetails{
		Format:            model.Config.ModelFormat,
		Family:            model.Config.ModelFamily,
		Families:          model.Config.ModelFamilies,
		ParameterSize:     model.Config.ModelType,
		QuantizationLevel: model.Config.FileType,
	}

651
652
653
654
655
656
657
658
	if req.System != "" {
		model.System = req.System
	}

	if req.Template != "" {
		model.Template = req.Template
	}

Patrick Devine's avatar
Patrick Devine committed
659
660
661
662
	resp := &api.ShowResponse{
		License:  strings.Join(model.License, "\n"),
		System:   model.System,
		Template: model.Template,
Patrick Devine's avatar
Patrick Devine committed
663
		Details:  modelDetails,
Patrick Devine's avatar
Patrick Devine committed
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
	}

	var params []string
	cs := 30
	for k, v := range model.Options {
		switch val := v.(type) {
		case string:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
		case int:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
		case float64:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
		case bool:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
		case []interface{}:
			for _, nv := range val {
				switch nval := nv.(type) {
				case string:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
				case int:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
				case float64:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
				case bool:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
				}
			}
		}
	}
	resp.Parameters = strings.Join(params, "\n")

695
696
697
698
699
700
701
702
703
704
705
706
707
	for k, v := range req.Options {
		if _, ok := req.Options[k]; ok {
			model.Options[k] = v
		}
	}

	mf, err := ShowModelfile(model)
	if err != nil {
		return nil, err
	}

	resp.Modelfile = mf

Patrick Devine's avatar
Patrick Devine committed
708
709
710
	return resp, nil
}

711
func ListModelsHandler(c *gin.Context) {
712
	models := make([]api.ModelResponse, 0)
Patrick Devine's avatar
Patrick Devine committed
713
714
715
716
717
	fp, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
718

Patrick Devine's avatar
Patrick Devine committed
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
	modelResponse := func(modelName string) (api.ModelResponse, error) {
		model, err := GetModel(modelName)
		if err != nil {
			return api.ModelResponse{}, err
		}

		modelDetails := api.ModelDetails{
			Format:            model.Config.ModelFormat,
			Family:            model.Config.ModelFamily,
			Families:          model.Config.ModelFamilies,
			ParameterSize:     model.Config.ModelType,
			QuantizationLevel: model.Config.FileType,
		}

		return api.ModelResponse{
			Name:    model.ShortName,
			Size:    model.Size,
			Digest:  model.Digest,
			Details: modelDetails,
		}, nil
	}

Michael Yang's avatar
Michael Yang committed
741
	walkFunc := func(path string, info os.FileInfo, _ error) error {
Patrick Devine's avatar
Patrick Devine committed
742
		if !info.IsDir() {
Michael Yang's avatar
Michael Yang committed
743
744
745
			dir, file := filepath.Split(path)
			dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
			tag := strings.Join([]string{dir, file}, ":")
746

Patrick Devine's avatar
Patrick Devine committed
747
			resp, err := modelResponse(tag)
Patrick Devine's avatar
Patrick Devine committed
748
			if err != nil {
749
750
				log.Printf("skipping file: %s", fp)
				return nil
Patrick Devine's avatar
Patrick Devine committed
751
			}
Michael Yang's avatar
Michael Yang committed
752

Patrick Devine's avatar
Patrick Devine committed
753
754
			resp.ModifiedAt = info.ModTime()
			models = append(models, resp)
Patrick Devine's avatar
Patrick Devine committed
755
		}
Michael Yang's avatar
Michael Yang committed
756

Patrick Devine's avatar
Patrick Devine committed
757
		return nil
Michael Yang's avatar
Michael Yang committed
758
759
760
	}

	if err := filepath.Walk(fp, walkFunc); err != nil {
Patrick Devine's avatar
Patrick Devine committed
761
762
763
764
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
765
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
766
767
}

Patrick Devine's avatar
Patrick Devine committed
768
769
func CopyModelHandler(c *gin.Context) {
	var req api.CopyRequest
Michael Yang's avatar
Michael Yang committed
770
771
772
773
774
775
776
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
777
778
779
		return
	}

780
781
782
783
784
	if req.Source == "" || req.Destination == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "source add destination are required"})
		return
	}

785
786
787
788
789
	if err := ParseModelPath(req.Destination).Validate(); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
790
791
792
793
794
795
796
797
798
799
	if err := CopyModel(req.Source, req.Destination); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}
}

Michael Yang's avatar
Michael Yang committed
800
func HeadBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
801
802
803
804
805
806
807
808
809
810
811
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); err != nil {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	}

Michael Yang's avatar
Michael Yang committed
812
	c.Status(http.StatusOK)
Michael Yang's avatar
Michael Yang committed
813
814
815
}

func CreateBlobHandler(c *gin.Context) {
816
	layer, err := NewLayer(c.Request.Body, "")
Michael Yang's avatar
Michael Yang committed
817
818
819
820
821
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

822
823
	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
Michael Yang's avatar
Michael Yang committed
824
825
826
		return
	}

827
	if _, err := layer.Commit(); err != nil {
Michael Yang's avatar
Michael Yang committed
828
829
830
831
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
832
	c.Status(http.StatusCreated)
Michael Yang's avatar
Michael Yang committed
833
834
}

Michael Yang's avatar
Michael Yang committed
835
836
837
838
839
840
var defaultAllowOrigins = []string{
	"localhost",
	"127.0.0.1",
	"0.0.0.0",
}

841
842
843
844
845
func NewServer() (*Server, error) {
	workDir, err := os.MkdirTemp("", "ollama")
	if err != nil {
		return nil, err
	}
846

847
848
849
850
	return &Server{
		WorkDir: workDir,
	}, nil
}
851

852
853
854
855
func (s *Server) GenerateRoutes() http.Handler {
	var origins []string
	if o := os.Getenv("OLLAMA_ORIGINS"); o != "" {
		origins = strings.Split(o, ",")
856
857
	}

Michael Yang's avatar
Michael Yang committed
858
859
	config := cors.DefaultConfig()
	config.AllowWildcard = true
860
	config.AllowBrowserExtensions = true
Michael Yang's avatar
Michael Yang committed
861

862
	config.AllowOrigins = origins
Michael Yang's avatar
Michael Yang committed
863
864
865
866
867
868
869
870
	for _, allowOrigin := range defaultAllowOrigins {
		config.AllowOrigins = append(config.AllowOrigins,
			fmt.Sprintf("http://%s", allowOrigin),
			fmt.Sprintf("https://%s", allowOrigin),
			fmt.Sprintf("http://%s:*", allowOrigin),
			fmt.Sprintf("https://%s:*", allowOrigin),
		)
	}
Michael Yang's avatar
Michael Yang committed
871

Bruce MacDonald's avatar
Bruce MacDonald committed
872
	r := gin.Default()
873
874
875
	r.Use(
		cors.New(config),
		func(c *gin.Context) {
876
			c.Set("workDir", s.WorkDir)
877
878
879
			c.Next()
		},
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
880

881
882
	r.POST("/api/pull", PullModelHandler)
	r.POST("/api/generate", GenerateHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
883
	r.POST("/api/chat", ChatHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
884
	r.POST("/api/embeddings", EmbeddingHandler)
885
886
	r.POST("/api/create", CreateModelHandler)
	r.POST("/api/push", PushModelHandler)
Patrick Devine's avatar
Patrick Devine committed
887
	r.POST("/api/copy", CopyModelHandler)
888
	r.DELETE("/api/delete", DeleteModelHandler)
Patrick Devine's avatar
Patrick Devine committed
889
	r.POST("/api/show", ShowModelHandler)
Michael Yang's avatar
Michael Yang committed
890
	r.POST("/api/blobs/:digest", CreateBlobHandler)
Michael Yang's avatar
Michael Yang committed
891
	r.HEAD("/api/blobs/:digest", HeadBlobHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
892

Michael Yang's avatar
Michael Yang committed
893
894
895
896
897
898
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

		r.Handle(method, "/api/tags", ListModelsHandler)
Michael Yang's avatar
Michael Yang committed
899
900
901
		r.Handle(method, "/api/version", func(c *gin.Context) {
			c.JSON(http.StatusOK, gin.H{"version": version.Version})
		})
Michael Yang's avatar
Michael Yang committed
902
903
	}

904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
	return r
}

func Serve(ln net.Listener) error {
	if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		// clean up unused layers and manifests
		if err := PruneLayers(); err != nil {
			return err
		}

		manifestsPath, err := GetManifestPath()
		if err != nil {
			return err
		}

		if err := PruneDirectory(manifestsPath); err != nil {
			return err
		}
	}

	s, err := NewServer()
	if err != nil {
		return err
	}
	r := s.GenerateRoutes()

Michael Yang's avatar
Michael Yang committed
930
	log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version)
931
	srvr := &http.Server{
Jeffrey Morgan's avatar
Jeffrey Morgan committed
932
933
934
		Handler: r,
	}

935
936
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
937
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
938
939
	go func() {
		<-signals
940
941
		if loaded.runner != nil {
			loaded.runner.Close()
942
		}
943
		os.RemoveAll(s.WorkDir)
944
945
946
		os.Exit(0)
	}()

947
948
949
950
	if err := llm.Init(s.WorkDir); err != nil {
		return fmt.Errorf("unable to initialize llm library %w", err)
	}
	if runtime.GOOS == "linux" { // TODO - windows too
951
		// check compatibility to log warnings
952
		if _, err := gpu.CheckVRAM(); err != nil {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
953
			log.Print(err.Error())
954
955
956
		}
	}

957
	return srvr.Serve(ln)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
958
}
Michael Yang's avatar
Michael Yang committed
959

960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
			if errorMsg, ok := r["error"].(string); ok {
				c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
				return
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in progress response"})
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
985
func streamResponse(c *gin.Context, ch chan any) {
986
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
987
988
989
990
991
992
993
994
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
995
			log.Printf("streamResponse: json.Marshal failed with %s", err)
Michael Yang's avatar
Michael Yang committed
996
997
998
			return false
		}

999
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
1000
1001
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1002
			log.Printf("streamResponse: w.Write failed with %s", err)
Michael Yang's avatar
Michael Yang committed
1003
1004
1005
1006
1007
1008
			return false
		}

		return true
	})
}
Bruce MacDonald's avatar
Bruce MacDonald committed
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036

func ChatHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()

	var req api.ChatRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	// validate the request
	switch {
	case req.Model == "":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	case len(req.Format) > 0 && req.Format != "json":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
		return
	}

1037
	model, err := GetModel(req.Model)
Bruce MacDonald's avatar
Bruce MacDonald committed
1038
1039
	if err != nil {
		var pErr *fs.PathError
1040
		if errors.As(err, &pErr) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1041
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
1042
1043
1044
1045
1046
1047
1048
1049
1050
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	opts, err := modelOptions(model, req.Options)
	if err != nil {
		if errors.Is(err, api.ErrInvalidOpts) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1051
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
1052
			return
Bruce MacDonald's avatar
Bruce MacDonald committed
1053
		}
1054
1055
1056
1057
1058
1059
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	sessionDuration := defaultSessionDuration
	if err := load(c, model, opts, sessionDuration); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
1060
1061
1062
1063
1064
		return
	}

	// an empty request loads the model
	if len(req.Messages) == 0 {
1065
		c.JSON(http.StatusOK, api.ChatResponse{CreatedAt: time.Now().UTC(), Model: req.Model, Done: true, Message: api.Message{Role: "assistant"}})
Bruce MacDonald's avatar
Bruce MacDonald committed
1066
1067
1068
1069
1070
		return
	}

	checkpointLoaded := time.Now()

1071
	prompt, images, err := model.ChatPrompt(req.Messages)
Bruce MacDonald's avatar
Bruce MacDonald committed
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	ch := make(chan any)

	go func() {
		defer close(ch)

		fn := func(r llm.PredictResult) {
			// Update model expiration
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)

			resp := api.ChatResponse{
1088
				Model:     req.Model,
1089
				CreatedAt: time.Now().UTC(),
1090
				Message:   api.Message{Role: "assistant", Content: r.Content},
Bruce MacDonald's avatar
Bruce MacDonald committed
1091
1092
1093
1094
1095
1096
1097
1098
1099
				Done:      r.Done,
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
			}

1100
1101
1102
			if r.Done {
				resp.TotalDuration = time.Since(checkpointStart)
				resp.LoadDuration = checkpointLoaded.Sub(checkpointStart)
Bruce MacDonald's avatar
Bruce MacDonald committed
1103
1104
1105
1106
1107
1108
1109
			}

			ch <- resp
		}

		// Start prediction
		predictReq := llm.PredictOpts{
1110
1111
1112
1113
			Prompt:  prompt,
			Format:  req.Format,
			Images:  images,
			Options: opts,
Bruce MacDonald's avatar
Bruce MacDonald committed
1114
1115
1116
1117
1118
1119
1120
		}
		if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
1121
1122
		// Accumulate responses into the final response
		var final api.ChatResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
1123
1124
		var sb strings.Builder
		for resp := range ch {
1125
1126
			switch r := resp.(type) {
			case api.ChatResponse:
1127
				sb.WriteString(r.Message.Content)
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
				final = r
			case gin.H:
				if errorMsg, ok := r["error"].(string); ok {
					c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
					return
				} else {
					c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
					return
				}
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
1140
1141
			}
		}
1142

1143
		final.Message = api.Message{Role: "assistant", Content: sb.String()}
1144
		c.JSON(http.StatusOK, final)
Bruce MacDonald's avatar
Bruce MacDonald committed
1145
1146
1147
1148
1149
		return
	}

	streamResponse(c, ch)
}