routes.go 27.9 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
4
	"context"
Michael Yang's avatar
Michael Yang committed
5
	"encoding/json"
6
	"errors"
7
	"fmt"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
8
	"io"
9
	"io/fs"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
10
11
12
	"log"
	"net"
	"net/http"
13
	"os"
14
	"os/signal"
Michael Yang's avatar
Michael Yang committed
15
	"path/filepath"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
16
	"reflect"
17
	"runtime"
Patrick Devine's avatar
Patrick Devine committed
18
	"strconv"
Michael Yang's avatar
Michael Yang committed
19
	"strings"
Michael Yang's avatar
Michael Yang committed
20
	"sync"
21
	"syscall"
22
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
23

Michael Yang's avatar
Michael Yang committed
24
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
25
26
	"github.com/gin-gonic/gin"

Jeffrey Morgan's avatar
Jeffrey Morgan committed
27
	"github.com/jmorganca/ollama/api"
28
	"github.com/jmorganca/ollama/gpu"
29
	"github.com/jmorganca/ollama/llm"
Michael Yang's avatar
Michael Yang committed
30
	"github.com/jmorganca/ollama/parser"
Michael Yang's avatar
Michael Yang committed
31
	"github.com/jmorganca/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
32
33
)

Michael Yang's avatar
Michael Yang committed
34
35
var mode string = gin.DebugMode

36
37
38
39
type Server struct {
	WorkDir string
}

Michael Yang's avatar
Michael Yang committed
40
41
42
43
44
45
46
47
48
49
50
51
func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
52
var loaded struct {
Michael Yang's avatar
Michael Yang committed
53
54
	mu sync.Mutex

55
	runner llm.LLM
Michael Yang's avatar
Michael Yang committed
56
57
58

	expireAt    time.Time
	expireTimer *time.Timer
Jeffrey Morgan's avatar
Jeffrey Morgan committed
59

60
61
	*Model
	*api.Options
Michael Yang's avatar
Michael Yang committed
62
63
}

64
65
var defaultSessionDuration = 5 * time.Minute

Bruce MacDonald's avatar
Bruce MacDonald committed
66
// load a model into memory if it is not already loaded, it is up to the caller to lock loaded.mu before calling this function
67
func load(c *gin.Context, model *Model, opts api.Options, sessionDuration time.Duration) error {
Bruce MacDonald's avatar
Bruce MacDonald committed
68
69
	workDir := c.GetString("workDir")

70
71
72
73
74
75
76
	needLoad := loaded.runner == nil || // is there a model loaded?
		loaded.ModelPath != model.ModelPath || // has the base model changed?
		!reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
		!reflect.DeepEqual(loaded.Options.Runner, opts.Runner) // have the runner options changed?

	if needLoad {
		if loaded.runner != nil {
77
			log.Println("changing loaded model")
78
79
80
81
			loaded.runner.Close()
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
82
		}
Michael Yang's avatar
Michael Yang committed
83

Michael Yang's avatar
Michael Yang committed
84
		llmRunner, err := llm.New(workDir, model.ModelPath, model.AdapterPaths, model.ProjectorPaths, opts)
Michael Yang's avatar
Michael Yang committed
85
		if err != nil {
86
87
88
			// some older models are not compatible with newer versions of llama.cpp
			// show a generalized compatibility error until there is a better way to
			// check for model compatibility
Bruce MacDonald's avatar
Bruce MacDonald committed
89
			if errors.Is(llm.ErrUnsupportedFormat, err) || strings.Contains(err.Error(), "failed to load model") {
90
91
92
				err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
			}

93
			return err
Michael Yang's avatar
Michael Yang committed
94
95
		}

96
97
98
		loaded.Model = model
		loaded.runner = llmRunner
		loaded.Options = &opts
Michael Yang's avatar
Michael Yang committed
99
	}
100

Jeffrey Morgan's avatar
Jeffrey Morgan committed
101
	loaded.expireAt = time.Now().Add(sessionDuration)
Bruce MacDonald's avatar
Bruce MacDonald committed
102

Jeffrey Morgan's avatar
Jeffrey Morgan committed
103
104
105
106
	if loaded.expireTimer == nil {
		loaded.expireTimer = time.AfterFunc(sessionDuration, func() {
			loaded.mu.Lock()
			defer loaded.mu.Unlock()
Michael Yang's avatar
Michael Yang committed
107

Jeffrey Morgan's avatar
Jeffrey Morgan committed
108
			if time.Now().Before(loaded.expireAt) {
Michael Yang's avatar
Michael Yang committed
109
110
111
				return
			}

112
113
			if loaded.runner != nil {
				loaded.runner.Close()
Michael Yang's avatar
Michael Yang committed
114
115
			}

116
117
118
			loaded.runner = nil
			loaded.Model = nil
			loaded.Options = nil
Michael Yang's avatar
Michael Yang committed
119
		})
Michael Yang's avatar
Michael Yang committed
120
	}
121

Jeffrey Morgan's avatar
Jeffrey Morgan committed
122
	loaded.expireTimer.Reset(sessionDuration)
123
124
125
126
127
128
129
130
131
132
133
134
135
136
	return nil
}

func modelOptions(model *Model, requestOpts map[string]interface{}) (api.Options, error) {
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		return api.Options{}, err
	}

	if err := opts.FromMap(requestOpts); err != nil {
		return api.Options{}, err
	}

	return opts, nil
Bruce MacDonald's avatar
Bruce MacDonald committed
137
138
139
140
141
142
143
144
}

func GenerateHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()
	var req api.GenerateRequest
Michael Yang's avatar
Michael Yang committed
145
	err := c.ShouldBindJSON(&req)
Patrick Devine's avatar
Patrick Devine committed
146

Michael Yang's avatar
Michael Yang committed
147
148
149
150
151
152
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
153
154
155
		return
	}

156
157
158
	// validate the request
	switch {
	case req.Model == "":
159
160
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
161
162
163
	case len(req.Format) > 0 && req.Format != "json":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
		return
164
165
166
	case req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
		return
167
168
	}

169
	model, err := GetModel(req.Model)
Bruce MacDonald's avatar
Bruce MacDonald committed
170
	if err != nil {
171
		var pErr *fs.PathError
172
		if errors.As(err, &pErr) {
173
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
174
175
176
177
178
179
180
181
182
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	opts, err := modelOptions(model, req.Options)
	if err != nil {
		if errors.Is(err, api.ErrInvalidOpts) {
Bruce MacDonald's avatar
Bruce MacDonald committed
183
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
184
			return
185
		}
186
187
188
189
190
191
192
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	sessionDuration := defaultSessionDuration
	if err := load(c, model, opts, sessionDuration); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
193
194
195
		return
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
196
197
	// an empty request loads the model
	if req.Prompt == "" && req.Template == "" && req.System == "" {
198
		c.JSON(http.StatusOK, api.GenerateResponse{
199
200
201
			CreatedAt: time.Now().UTC(),
			Model:     req.Model,
			Done:      true})
Bruce MacDonald's avatar
Bruce MacDonald committed
202
203
204
205
206
		return
	}

	checkpointLoaded := time.Now()

Bruce MacDonald's avatar
Bruce MacDonald committed
207
	var prompt string
208
	var promptVars PromptVars
Bruce MacDonald's avatar
Bruce MacDonald committed
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
	switch {
	case req.Raw:
		prompt = req.Prompt
	case req.Prompt != "":
		if req.Template != "" {
			// override the default model template
			model.Template = req.Template
		}

		var rebuild strings.Builder
		if req.Context != nil {
			// TODO: context is deprecated, at some point the context logic within this conditional should be removed
			prevCtx, err := loaded.runner.Decode(c.Request.Context(), req.Context)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}

			// Remove leading spaces from prevCtx if present
			prevCtx = strings.TrimPrefix(prevCtx, " ")
			rebuild.WriteString(prevCtx)
		}
231
		promptVars = PromptVars{
Bruce MacDonald's avatar
Bruce MacDonald committed
232
233
234
			System: req.System,
			Prompt: req.Prompt,
			First:  len(req.Context) == 0,
235
236
		}
		p, err := model.PreResponsePrompt(promptVars)
237
238
239
240
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
241
242
		rebuild.WriteString(p)
		prompt = rebuild.String()
Bruce MacDonald's avatar
Bruce MacDonald committed
243
244
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
245
	ch := make(chan any)
Bruce MacDonald's avatar
Bruce MacDonald committed
246
	var generated strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
247
248
249
	go func() {
		defer close(ch)

Bruce MacDonald's avatar
Bruce MacDonald committed
250
251
		fn := func(r llm.PredictResult) {
			// Update model expiration
Bruce MacDonald's avatar
Bruce MacDonald committed
252
253
254
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)

Bruce MacDonald's avatar
Bruce MacDonald committed
255
256
257
258
			// Build up the full response
			if _, err := generated.WriteString(r.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
259
260
			}

Bruce MacDonald's avatar
Bruce MacDonald committed
261
			resp := api.GenerateResponse{
262
				Model:     req.Model,
263
				CreatedAt: time.Now().UTC(),
264
265
				Done:      r.Done,
				Response:  r.Content,
Bruce MacDonald's avatar
Bruce MacDonald committed
266
267
268
269
270
271
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
Bruce MacDonald's avatar
Bruce MacDonald committed
272
273
			}

274
275
276
277
278
			if r.Done {
				resp.TotalDuration = time.Since(checkpointStart)
				resp.LoadDuration = checkpointLoaded.Sub(checkpointStart)

				if !req.Raw {
279
280
281
282
283
284
285
286
					// append the generated text to the history and template it if needed
					promptVars.Response = generated.String()
					result, err := model.PostResponseTemplate(promptVars)
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
					embd, err := loaded.runner.Encode(c.Request.Context(), prompt+result)
287
288
289
290
291
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
					resp.Context = embd
Bruce MacDonald's avatar
Bruce MacDonald committed
292
293
294
295
				}
			}

			ch <- resp
Bruce MacDonald's avatar
Bruce MacDonald committed
296
297
		}

Bruce MacDonald's avatar
Bruce MacDonald committed
298
299
		// Start prediction
		predictReq := llm.PredictOpts{
300
301
302
303
			Prompt:  prompt,
			Format:  req.Format,
			Images:  req.Images,
			Options: opts,
Bruce MacDonald's avatar
Bruce MacDonald committed
304
305
		}
		if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
306
307
308
309
310
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
311
312
		// Accumulate responses into the final response
		var final api.GenerateResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
313
		var sb strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
314
		for resp := range ch {
315
316
317
318
319
320
321
322
323
324
325
326
327
328
			switch r := resp.(type) {
			case api.GenerateResponse:
				sb.WriteString(r.Response)
				final = r
			case gin.H:
				if errorMsg, ok := r["error"].(string); ok {
					c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
					return
				} else {
					c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
					return
				}
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
Bruce MacDonald's avatar
Bruce MacDonald committed
329
330
331
				return
			}
		}
332
333
334

		final.Response = sb.String()
		c.JSON(http.StatusOK, final)
Bruce MacDonald's avatar
Bruce MacDonald committed
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
		return
	}

	streamResponse(c, ch)
}

func EmbeddingHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	var req api.EmbeddingRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if req.Model == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

361
	model, err := GetModel(req.Model)
Bruce MacDonald's avatar
Bruce MacDonald committed
362
	if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
363
		var pErr *fs.PathError
364
		if errors.As(err, &pErr) {
Bruce MacDonald's avatar
Bruce MacDonald committed
365
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
366
367
368
369
370
371
372
373
374
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	opts, err := modelOptions(model, req.Options)
	if err != nil {
		if errors.Is(err, api.ErrInvalidOpts) {
Bruce MacDonald's avatar
Bruce MacDonald committed
375
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
376
			return
Bruce MacDonald's avatar
Bruce MacDonald committed
377
		}
378
379
380
381
382
383
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	sessionDuration := defaultSessionDuration
	if err := load(c, model, opts, sessionDuration); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
384
385
386
		return
	}

387
	if !loaded.Options.EmbeddingOnly {
Bruce MacDonald's avatar
Bruce MacDonald committed
388
389
390
391
		c.JSON(http.StatusBadRequest, gin.H{"error": "embedding option must be set to true"})
		return
	}

392
	embedding, err := loaded.runner.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
393
394
395
396
397
398
399
400
401
402
403
404
	if err != nil {
		log.Printf("embedding generation failed: %v", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to generate embedding"})
		return
	}

	resp := api.EmbeddingResponse{
		Embedding: embedding,
	}
	c.JSON(http.StatusOK, resp)
}

405
func PullModelHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
406
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
407
408
409
410
411
412
413
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
414
415
416
		return
	}

417
418
419
420
421
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

422
423
424
	ch := make(chan any)
	go func() {
		defer close(ch)
425
426
		fn := func(r api.ProgressResponse) {
			ch <- r
427
		}
428

429
430
431
432
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

433
434
435
436
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

		if err := PullModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
437
			ch <- gin.H{"error": err.Error()}
438
439
440
		}
	}()

441
442
443
444
445
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

446
447
448
	streamResponse(c, ch)
}

449
func PushModelHandler(c *gin.Context) {
450
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
451
452
453
454
455
456
457
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
458
459
		return
	}
Michael Yang's avatar
Michael Yang committed
460

461
462
463
464
465
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

466
467
468
	ch := make(chan any)
	go func() {
		defer close(ch)
469
470
		fn := func(r api.ProgressResponse) {
			ch <- r
471
		}
472

473
474
475
476
		regOpts := &RegistryOptions{
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
477
478
479
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

480
		if err := PushModel(ctx, req.Name, regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
481
			ch <- gin.H{"error": err.Error()}
482
483
484
		}
	}()

485
486
487
488
489
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

490
491
492
	streamResponse(c, ch)
}

493
func CreateModelHandler(c *gin.Context) {
494
	var req api.CreateRequest
Michael Yang's avatar
Michael Yang committed
495
496
497
498
499
500
501
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
502
		return
503
504
	}

Michael Yang's avatar
Michael Yang committed
505
506
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
507
508
509
		return
	}

510
511
	if err := ParseModelPath(req.Name).Validate(); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
512
513
514
		return
	}

Michael Yang's avatar
Michael Yang committed
515
516
	if req.Path == "" && req.Modelfile == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "path or modelfile are required"})
Michael Yang's avatar
Michael Yang committed
517
518
		return
	}
Michael Yang's avatar
Michael Yang committed
519
520
521

	var modelfile io.Reader = strings.NewReader(req.Modelfile)
	if req.Path != "" && req.Modelfile == "" {
522
		mf, err := os.Open(req.Path)
Michael Yang's avatar
Michael Yang committed
523
524
525
526
		if err != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("error reading modelfile: %s", err)})
			return
		}
527
		defer mf.Close()
Michael Yang's avatar
Michael Yang committed
528

529
		modelfile = mf
Michael Yang's avatar
Michael Yang committed
530
	}
Michael Yang's avatar
Michael Yang committed
531
532
533
534
535
536
537

	commands, err := parser.Parse(modelfile)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
538
	ch := make(chan any)
Michael Yang's avatar
Michael Yang committed
539
540
	go func() {
		defer close(ch)
541
542
		fn := func(resp api.ProgressResponse) {
			ch <- resp
543
544
		}

545
546
547
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

548
		if err := CreateModel(ctx, req.Name, filepath.Dir(req.Path), commands, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
549
			ch <- gin.H{"error": err.Error()}
550
		}
Michael Yang's avatar
Michael Yang committed
551
	}()
Michael Yang's avatar
Michael Yang committed
552

553
554
555
556
557
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

Michael Yang's avatar
Michael Yang committed
558
	streamResponse(c, ch)
Bruce MacDonald's avatar
Bruce MacDonald committed
559
560
}

561
562
func DeleteModelHandler(c *gin.Context) {
	var req api.DeleteRequest
Michael Yang's avatar
Michael Yang committed
563
564
565
566
567
568
569
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
570
571
572
		return
	}

573
574
575
576
577
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

578
579
580
581
	if err := DeleteModel(req.Name); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
582
583
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
584
585
		return
	}
Michael Yang's avatar
Michael Yang committed
586
587
588
589
590
591
592
593
594
595
596
597

	manifestsPath, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	if err := PruneDirectory(manifestsPath); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

598
	c.JSON(http.StatusOK, nil)
599
600
}

Patrick Devine's avatar
Patrick Devine committed
601
602
func ShowModelHandler(c *gin.Context) {
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
603
604
605
606
607
608
609
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
610
611
612
		return
	}

613
614
615
616
617
	if req.Name == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "name is required"})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
	resp, err := GetModelInfo(req.Name)
	if err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Name)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

func GetModelInfo(name string) (*api.ShowResponse, error) {
	model, err := GetModel(name)
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
637
638
639
640
641
642
643
644
	modelDetails := api.ModelDetails{
		Format:            model.Config.ModelFormat,
		Family:            model.Config.ModelFamily,
		Families:          model.Config.ModelFamilies,
		ParameterSize:     model.Config.ModelType,
		QuantizationLevel: model.Config.FileType,
	}

Patrick Devine's avatar
Patrick Devine committed
645
646
647
648
	resp := &api.ShowResponse{
		License:  strings.Join(model.License, "\n"),
		System:   model.System,
		Template: model.Template,
Patrick Devine's avatar
Patrick Devine committed
649
		Details:  modelDetails,
Patrick Devine's avatar
Patrick Devine committed
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
	}

	mf, err := ShowModelfile(model)
	if err != nil {
		return nil, err
	}

	resp.Modelfile = mf

	var params []string
	cs := 30
	for k, v := range model.Options {
		switch val := v.(type) {
		case string:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, val))
		case int:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(val)))
		case float64:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(val, 'f', 0, 64)))
		case bool:
			params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(val)))
		case []interface{}:
			for _, nv := range val {
				switch nval := nv.(type) {
				case string:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, nval))
				case int:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.Itoa(nval)))
				case float64:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatFloat(nval, 'f', 0, 64)))
				case bool:
					params = append(params, fmt.Sprintf("%-*s %s", cs, k, strconv.FormatBool(nval)))
				}
			}
		}
	}
	resp.Parameters = strings.Join(params, "\n")

	return resp, nil
}

691
func ListModelsHandler(c *gin.Context) {
692
	models := make([]api.ModelResponse, 0)
Patrick Devine's avatar
Patrick Devine committed
693
694
695
696
697
	fp, err := GetManifestPath()
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
698

Patrick Devine's avatar
Patrick Devine committed
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
	modelResponse := func(modelName string) (api.ModelResponse, error) {
		model, err := GetModel(modelName)
		if err != nil {
			return api.ModelResponse{}, err
		}

		modelDetails := api.ModelDetails{
			Format:            model.Config.ModelFormat,
			Family:            model.Config.ModelFamily,
			Families:          model.Config.ModelFamilies,
			ParameterSize:     model.Config.ModelType,
			QuantizationLevel: model.Config.FileType,
		}

		return api.ModelResponse{
			Name:    model.ShortName,
			Size:    model.Size,
			Digest:  model.Digest,
			Details: modelDetails,
		}, nil
	}

Michael Yang's avatar
Michael Yang committed
721
	walkFunc := func(path string, info os.FileInfo, _ error) error {
Patrick Devine's avatar
Patrick Devine committed
722
		if !info.IsDir() {
Michael Yang's avatar
Michael Yang committed
723
724
725
			dir, file := filepath.Split(path)
			dir = strings.Trim(strings.TrimPrefix(dir, fp), string(os.PathSeparator))
			tag := strings.Join([]string{dir, file}, ":")
726

Patrick Devine's avatar
Patrick Devine committed
727
			resp, err := modelResponse(tag)
Patrick Devine's avatar
Patrick Devine committed
728
			if err != nil {
729
730
				log.Printf("skipping file: %s", fp)
				return nil
Patrick Devine's avatar
Patrick Devine committed
731
			}
Michael Yang's avatar
Michael Yang committed
732

Patrick Devine's avatar
Patrick Devine committed
733
734
			resp.ModifiedAt = info.ModTime()
			models = append(models, resp)
Patrick Devine's avatar
Patrick Devine committed
735
		}
Michael Yang's avatar
Michael Yang committed
736

Patrick Devine's avatar
Patrick Devine committed
737
		return nil
Michael Yang's avatar
Michael Yang committed
738
739
740
	}

	if err := filepath.Walk(fp, walkFunc); err != nil {
Patrick Devine's avatar
Patrick Devine committed
741
742
743
744
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
745
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
746
747
}

Patrick Devine's avatar
Patrick Devine committed
748
749
func CopyModelHandler(c *gin.Context) {
	var req api.CopyRequest
Michael Yang's avatar
Michael Yang committed
750
751
752
753
754
755
756
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
757
758
759
		return
	}

760
761
762
763
764
	if req.Source == "" || req.Destination == "" {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "source add destination are required"})
		return
	}

765
766
767
768
769
	if err := ParseModelPath(req.Destination).Validate(); err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Patrick Devine's avatar
Patrick Devine committed
770
771
772
773
774
775
776
777
778
779
	if err := CopyModel(req.Source, req.Destination); err != nil {
		if os.IsNotExist(err) {
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Source)})
		} else {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}
}

Michael Yang's avatar
Michael Yang committed
780
func HeadBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
781
782
783
784
785
786
787
788
789
790
791
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); err != nil {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	}

Michael Yang's avatar
Michael Yang committed
792
	c.Status(http.StatusOK)
Michael Yang's avatar
Michael Yang committed
793
794
795
}

func CreateBlobHandler(c *gin.Context) {
796
	layer, err := NewLayer(c.Request.Body, "")
Michael Yang's avatar
Michael Yang committed
797
798
799
800
801
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

802
803
	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
Michael Yang's avatar
Michael Yang committed
804
805
806
		return
	}

807
	if _, err := layer.Commit(); err != nil {
Michael Yang's avatar
Michael Yang committed
808
809
810
811
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
812
	c.Status(http.StatusCreated)
Michael Yang's avatar
Michael Yang committed
813
814
}

Michael Yang's avatar
Michael Yang committed
815
816
817
818
819
820
var defaultAllowOrigins = []string{
	"localhost",
	"127.0.0.1",
	"0.0.0.0",
}

821
822
823
824
825
func NewServer() (*Server, error) {
	workDir, err := os.MkdirTemp("", "ollama")
	if err != nil {
		return nil, err
	}
826

827
828
829
830
	return &Server{
		WorkDir: workDir,
	}, nil
}
831

832
833
834
835
func (s *Server) GenerateRoutes() http.Handler {
	var origins []string
	if o := os.Getenv("OLLAMA_ORIGINS"); o != "" {
		origins = strings.Split(o, ",")
836
837
	}

Michael Yang's avatar
Michael Yang committed
838
839
	config := cors.DefaultConfig()
	config.AllowWildcard = true
840
	config.AllowBrowserExtensions = true
Michael Yang's avatar
Michael Yang committed
841

842
	config.AllowOrigins = origins
Michael Yang's avatar
Michael Yang committed
843
844
845
846
847
848
849
850
	for _, allowOrigin := range defaultAllowOrigins {
		config.AllowOrigins = append(config.AllowOrigins,
			fmt.Sprintf("http://%s", allowOrigin),
			fmt.Sprintf("https://%s", allowOrigin),
			fmt.Sprintf("http://%s:*", allowOrigin),
			fmt.Sprintf("https://%s:*", allowOrigin),
		)
	}
Michael Yang's avatar
Michael Yang committed
851

Bruce MacDonald's avatar
Bruce MacDonald committed
852
	r := gin.Default()
853
854
855
	r.Use(
		cors.New(config),
		func(c *gin.Context) {
856
			c.Set("workDir", s.WorkDir)
857
858
859
			c.Next()
		},
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
860

861
862
	r.POST("/api/pull", PullModelHandler)
	r.POST("/api/generate", GenerateHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
863
	r.POST("/api/chat", ChatHandler)
Bruce MacDonald's avatar
Bruce MacDonald committed
864
	r.POST("/api/embeddings", EmbeddingHandler)
865
866
	r.POST("/api/create", CreateModelHandler)
	r.POST("/api/push", PushModelHandler)
Patrick Devine's avatar
Patrick Devine committed
867
	r.POST("/api/copy", CopyModelHandler)
868
	r.DELETE("/api/delete", DeleteModelHandler)
Patrick Devine's avatar
Patrick Devine committed
869
	r.POST("/api/show", ShowModelHandler)
Michael Yang's avatar
Michael Yang committed
870
	r.POST("/api/blobs/:digest", CreateBlobHandler)
Michael Yang's avatar
Michael Yang committed
871
	r.HEAD("/api/blobs/:digest", HeadBlobHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
872

Michael Yang's avatar
Michael Yang committed
873
874
875
876
877
878
	for _, method := range []string{http.MethodGet, http.MethodHead} {
		r.Handle(method, "/", func(c *gin.Context) {
			c.String(http.StatusOK, "Ollama is running")
		})

		r.Handle(method, "/api/tags", ListModelsHandler)
Michael Yang's avatar
Michael Yang committed
879
880
881
		r.Handle(method, "/api/version", func(c *gin.Context) {
			c.JSON(http.StatusOK, gin.H{"version": version.Version})
		})
Michael Yang's avatar
Michael Yang committed
882
883
	}

884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
	return r
}

func Serve(ln net.Listener) error {
	if noprune := os.Getenv("OLLAMA_NOPRUNE"); noprune == "" {
		// clean up unused layers and manifests
		if err := PruneLayers(); err != nil {
			return err
		}

		manifestsPath, err := GetManifestPath()
		if err != nil {
			return err
		}

		if err := PruneDirectory(manifestsPath); err != nil {
			return err
		}
	}

	s, err := NewServer()
	if err != nil {
		return err
	}
	r := s.GenerateRoutes()

Michael Yang's avatar
Michael Yang committed
910
	log.Printf("Listening on %s (version %s)", ln.Addr(), version.Version)
911
	srvr := &http.Server{
Jeffrey Morgan's avatar
Jeffrey Morgan committed
912
913
914
		Handler: r,
	}

915
916
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
917
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
918
919
	go func() {
		<-signals
920
921
		if loaded.runner != nil {
			loaded.runner.Close()
922
		}
923
		os.RemoveAll(s.WorkDir)
924
925
926
		os.Exit(0)
	}()

927
928
929
930
	if err := llm.Init(s.WorkDir); err != nil {
		return fmt.Errorf("unable to initialize llm library %w", err)
	}
	if runtime.GOOS == "linux" { // TODO - windows too
931
		// check compatibility to log warnings
932
		if _, err := gpu.CheckVRAM(); err != nil {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
933
			log.Print(err.Error())
934
935
936
		}
	}

937
	return srvr.Serve(ln)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
938
}
Michael Yang's avatar
Michael Yang committed
939

940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
func waitForStream(c *gin.Context, ch chan interface{}) {
	c.Header("Content-Type", "application/json")
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
			if r.Status == "success" {
				c.JSON(http.StatusOK, r)
				return
			}
		case gin.H:
			if errorMsg, ok := r["error"].(string); ok {
				c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
				return
			} else {
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in progress response"})
				return
			}
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected progress response"})
			return
		}
	}
	c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected end of progress response"})
}

Michael Yang's avatar
Michael Yang committed
965
func streamResponse(c *gin.Context, ch chan any) {
966
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
967
968
969
970
971
972
973
974
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

		bts, err := json.Marshal(val)
		if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
975
			log.Printf("streamResponse: json.Marshal failed with %s", err)
Michael Yang's avatar
Michael Yang committed
976
977
978
			return false
		}

979
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
980
981
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
982
			log.Printf("streamResponse: w.Write failed with %s", err)
Michael Yang's avatar
Michael Yang committed
983
984
985
986
987
988
			return false
		}

		return true
	})
}
Bruce MacDonald's avatar
Bruce MacDonald committed
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016

func ChatHandler(c *gin.Context) {
	loaded.mu.Lock()
	defer loaded.mu.Unlock()

	checkpointStart := time.Now()

	var req api.ChatRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	// validate the request
	switch {
	case req.Model == "":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	case len(req.Format) > 0 && req.Format != "json":
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "format must be json"})
		return
	}

1017
	model, err := GetModel(req.Model)
Bruce MacDonald's avatar
Bruce MacDonald committed
1018
1019
	if err != nil {
		var pErr *fs.PathError
1020
		if errors.As(err, &pErr) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1021
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found, try pulling it first", req.Model)})
1022
1023
1024
1025
1026
1027
1028
1029
1030
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	opts, err := modelOptions(model, req.Options)
	if err != nil {
		if errors.Is(err, api.ErrInvalidOpts) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1031
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
1032
			return
Bruce MacDonald's avatar
Bruce MacDonald committed
1033
		}
1034
1035
1036
1037
1038
1039
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
	sessionDuration := defaultSessionDuration
	if err := load(c, model, opts, sessionDuration); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
1040
1041
1042
1043
1044
		return
	}

	// an empty request loads the model
	if len(req.Messages) == 0 {
1045
		c.JSON(http.StatusOK, api.ChatResponse{CreatedAt: time.Now().UTC(), Model: req.Model, Done: true, Message: api.Message{Role: "assistant"}})
Bruce MacDonald's avatar
Bruce MacDonald committed
1046
1047
1048
1049
1050
		return
	}

	checkpointLoaded := time.Now()

1051
	prompt, images, err := model.ChatPrompt(req.Messages)
Bruce MacDonald's avatar
Bruce MacDonald committed
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	ch := make(chan any)

	go func() {
		defer close(ch)

		fn := func(r llm.PredictResult) {
			// Update model expiration
			loaded.expireAt = time.Now().Add(sessionDuration)
			loaded.expireTimer.Reset(sessionDuration)

			resp := api.ChatResponse{
1068
				Model:     req.Model,
1069
				CreatedAt: time.Now().UTC(),
1070
				Message:   api.Message{Role: "assistant", Content: r.Content},
Bruce MacDonald's avatar
Bruce MacDonald committed
1071
1072
1073
1074
1075
1076
1077
1078
1079
				Done:      r.Done,
				Metrics: api.Metrics{
					PromptEvalCount:    r.PromptEvalCount,
					PromptEvalDuration: r.PromptEvalDuration,
					EvalCount:          r.EvalCount,
					EvalDuration:       r.EvalDuration,
				},
			}

1080
1081
1082
			if r.Done {
				resp.TotalDuration = time.Since(checkpointStart)
				resp.LoadDuration = checkpointLoaded.Sub(checkpointStart)
Bruce MacDonald's avatar
Bruce MacDonald committed
1083
1084
1085
1086
1087
1088
1089
			}

			ch <- resp
		}

		// Start prediction
		predictReq := llm.PredictOpts{
1090
1091
1092
1093
			Prompt:  prompt,
			Format:  req.Format,
			Images:  images,
			Options: opts,
Bruce MacDonald's avatar
Bruce MacDonald committed
1094
1095
1096
1097
1098
1099
1100
		}
		if err := loaded.runner.Predict(c.Request.Context(), predictReq, fn); err != nil {
			ch <- gin.H{"error": err.Error()}
		}
	}()

	if req.Stream != nil && !*req.Stream {
1101
1102
		// Accumulate responses into the final response
		var final api.ChatResponse
Bruce MacDonald's avatar
Bruce MacDonald committed
1103
1104
		var sb strings.Builder
		for resp := range ch {
1105
1106
			switch r := resp.(type) {
			case api.ChatResponse:
1107
				sb.WriteString(r.Message.Content)
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
				final = r
			case gin.H:
				if errorMsg, ok := r["error"].(string); ok {
					c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
					return
				} else {
					c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
					return
				}
			default:
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
1120
1121
			}
		}
1122

1123
		final.Message = api.Message{Role: "assistant", Content: sb.String()}
1124
		c.JSON(http.StatusOK, final)
Bruce MacDonald's avatar
Bruce MacDonald committed
1125
1126
1127
1128
1129
		return
	}

	streamResponse(c, ch)
}