routes.go 64.1 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package server

import (
Michael Yang's avatar
Michael Yang committed
4
	"bytes"
Michael Yang's avatar
Michael Yang committed
5
	"cmp"
6
	"context"
7
	"encoding/base64"
Michael Yang's avatar
Michael Yang committed
8
	"encoding/json"
9
	"errors"
10
	"fmt"
11
	"image"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
12
	"io"
13
	"io/fs"
14
	"log/slog"
15
	"math"
Grace's avatar
Grace committed
16
	"math/rand"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
17
18
	"net"
	"net/http"
19
	"net/netip"
20
	"net/url"
21
	"os"
22
	"os/signal"
23
	"slices"
Michael Yang's avatar
Michael Yang committed
24
	"strings"
25
	"syscall"
26
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
27

Michael Yang's avatar
Michael Yang committed
28
	"github.com/gin-contrib/cors"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
29
	"github.com/gin-gonic/gin"
30
	"golang.org/x/image/webp"
31
	"golang.org/x/sync/errgroup"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
32

33
	"github.com/ollama/ollama/api"
34
	"github.com/ollama/ollama/auth"
35
	"github.com/ollama/ollama/discover"
36
	"github.com/ollama/ollama/envconfig"
37
	"github.com/ollama/ollama/format"
Michael Yang's avatar
Michael Yang committed
38
	"github.com/ollama/ollama/fs/ggml"
39
	"github.com/ollama/ollama/llm"
40
	"github.com/ollama/ollama/logutil"
41
	"github.com/ollama/ollama/middleware"
Devon Rifkin's avatar
Devon Rifkin committed
42
	"github.com/ollama/ollama/model/parsers"
43
	"github.com/ollama/ollama/model/renderers"
44
45
	"github.com/ollama/ollama/server/internal/client/ollama"
	"github.com/ollama/ollama/server/internal/registry"
Michael Yang's avatar
Michael Yang committed
46
	"github.com/ollama/ollama/template"
47
	"github.com/ollama/ollama/thinking"
48
	"github.com/ollama/ollama/tools"
49
	"github.com/ollama/ollama/types/errtypes"
Michael Yang's avatar
Michael Yang committed
50
	"github.com/ollama/ollama/types/model"
51
	"github.com/ollama/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
52
53
)

54
55
const signinURLStr = "https://ollama.com/connect?name=%s&key=%s"

56
57
58
59
60
61
62
63
64
65
66
67
func shouldUseHarmony(model *Model) bool {
	if slices.Contains([]string{"gptoss", "gpt-oss"}, model.Config.ModelFamily) {
		// heuristic to check whether the template expects to be parsed via harmony:
		// search for harmony tags that are nearly always used
		if model.Template.Contains("<|start|>") && model.Template.Contains("<|end|>") {
			return true
		}
	}

	return false
}

68
69
70
71
72
73
func experimentEnabled(name string) bool {
	return slices.Contains(strings.Split(os.Getenv("OLLAMA_EXPERIMENT"), ","), name)
}

var useClient2 = experimentEnabled("client2")

74
75
76
77
// Low VRAM mode is based on the sum of total VRAM (not free) and triggers
// reduced context length on some models
var lowVRAMThreshold uint64 = 20 * format.GibiByte

Michael Yang's avatar
Michael Yang committed
78
79
var mode string = gin.DebugMode

80
type Server struct {
81
82
83
	addr    net.Addr
	sched   *Scheduler
	lowVRAM bool
84
85
}

Michael Yang's avatar
Michael Yang committed
86
87
88
89
90
91
92
93
94
95
func init() {
	switch mode {
	case gin.DebugMode:
	case gin.ReleaseMode:
	case gin.TestMode:
	default:
		mode = gin.DebugMode
	}

	gin.SetMode(mode)
96
97
98

	// Tell renderers to use [img] tags
	renderers.RenderImgTags = true
Michael Yang's avatar
Michael Yang committed
99
100
}

Michael Yang's avatar
lint  
Michael Yang committed
101
102
103
104
var (
	errRequired    = errors.New("is required")
	errBadTemplate = errors.New("template error")
)
Michael Yang's avatar
Michael Yang committed
105

106
func modelOptions(model *Model, requestOpts map[string]any) (api.Options, error) {
107
108
109
110
111
112
113
114
115
116
	opts := api.DefaultOptions()
	if err := opts.FromMap(model.Options); err != nil {
		return api.Options{}, err
	}

	if err := opts.FromMap(requestOpts); err != nil {
		return api.Options{}, err
	}

	return opts, nil
Bruce MacDonald's avatar
Bruce MacDonald committed
117
118
}

Michael Yang's avatar
Michael Yang committed
119
120
// scheduleRunner schedules a runner after validating inputs such as capabilities and model options.
// It returns the allocated runner, model instance, and consolidated options if successful and error otherwise.
121
func (s *Server) scheduleRunner(ctx context.Context, name string, caps []model.Capability, requestOpts map[string]any, keepAlive *api.Duration) (llm.LlamaServer, *Model, *api.Options, error) {
Michael Yang's avatar
Michael Yang committed
122
	if name == "" {
Michael Yang's avatar
Michael Yang committed
123
		return nil, nil, nil, fmt.Errorf("model %w", errRequired)
Bruce MacDonald's avatar
Bruce MacDonald committed
124
125
	}

Michael Yang's avatar
Michael Yang committed
126
	model, err := GetModel(name)
Bruce MacDonald's avatar
Bruce MacDonald committed
127
	if err != nil {
Michael Yang's avatar
Michael Yang committed
128
		return nil, nil, nil, err
129
130
	}

131
132
133
134
	if slices.Contains(model.Config.ModelFamilies, "mllama") && len(model.ProjectorPaths) > 0 {
		return nil, nil, nil, fmt.Errorf("'llama3.2-vision' is no longer compatible with your version of Ollama and has been replaced by a newer version. To re-download, run 'ollama pull llama3.2-vision'")
	}

Michael Yang's avatar
Michael Yang committed
135
	if err := model.CheckCapabilities(caps...); err != nil {
Michael Yang's avatar
Michael Yang committed
136
		return nil, nil, nil, fmt.Errorf("%s %w", name, err)
137
138
	}

Michael Yang's avatar
Michael Yang committed
139
	opts, err := modelOptions(model, requestOpts)
140
	if err != nil {
Michael Yang's avatar
Michael Yang committed
141
		return nil, nil, nil, err
142
143
	}

144
145
	// This model is much more capable with a larger context, so set that
	// unless it would penalize performance too much
146
147
148
149
	if !s.lowVRAM && slices.Contains([]string{
		"gptoss", "gpt-oss",
		"qwen3vl", "qwen3vlmoe",
	}, model.Config.ModelFamily) {
Michael Yang's avatar
Michael Yang committed
150
151
152
		opts.NumCtx = max(opts.NumCtx, 8192)
	}

Michael Yang's avatar
Michael Yang committed
153
	runnerCh, errCh := s.sched.GetRunner(ctx, model, opts, keepAlive)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
154
155
	var runner *runnerRef
	select {
Michael Yang's avatar
Michael Yang committed
156
157
	case runner = <-runnerCh:
	case err = <-errCh:
Michael Yang's avatar
Michael Yang committed
158
		return nil, nil, nil, err
Bruce MacDonald's avatar
Bruce MacDonald committed
159
160
	}

Michael Yang's avatar
Michael Yang committed
161
	return runner.llama, model, &opts, nil
Michael Yang's avatar
Michael Yang committed
162
163
}

164
165
166
167
168
169
170
171
172
173
174
func signinURL() (string, error) {
	pubKey, err := auth.GetPublicKey()
	if err != nil {
		return "", err
	}

	encKey := base64.RawURLEncoding.EncodeToString([]byte(pubKey))
	h, _ := os.Hostname()
	return fmt.Sprintf(signinURLStr, url.PathEscape(h), encKey), nil
}

Michael Yang's avatar
Michael Yang committed
175
func (s *Server) GenerateHandler(c *gin.Context) {
176
	checkpointStart := time.Now()
Michael Yang's avatar
Michael Yang committed
177
178
179
180
181
182
	var req api.GenerateRequest
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	} else if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Bruce MacDonald's avatar
Bruce MacDonald committed
183
184
185
		return
	}

186
187
188
189
190
	if req.TopLogprobs < 0 || req.TopLogprobs > 20 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "top_logprobs must be between 0 and 20"})
		return
	}

191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
	name := model.ParseName(req.Model)
	if !name.IsValid() {
		// Ideally this is "invalid model name" but we're keeping with
		// what the API currently returns until we can change it.
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		return
	}

	// We cannot currently consolidate this into GetModel because all we'll
	// induce infinite recursion given the current code structure.
	name, err := getExistingName(name)
	if err != nil {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		return
	}

207
	m, err := GetModel(name.String())
208
209
	if err != nil {
		switch {
210
		case errors.Is(err, fs.ErrNotExist):
211
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
212
		case err.Error() == errtypes.InvalidModelNameErrMsg:
213
214
215
216
217
218
219
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

220
221
222
223
224
	if req.TopLogprobs < 0 || req.TopLogprobs > 20 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "top_logprobs must be between 0 and 20"})
		return
	}

225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
	if m.Config.RemoteHost != "" && m.Config.RemoteModel != "" {
		origModel := req.Model

		remoteURL, err := url.Parse(m.Config.RemoteHost)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		if !slices.Contains(envconfig.Remotes(), remoteURL.Hostname()) {
			slog.Info("remote model", "remotes", envconfig.Remotes(), "remoteURL", m.Config.RemoteHost, "hostname", remoteURL.Hostname())
			c.JSON(http.StatusBadRequest, gin.H{"error": "this server cannot run this remote model"})
			return
		}

		req.Model = m.Config.RemoteModel

		if req.Template == "" && m.Template.String() != "" {
			req.Template = m.Template.String()
		}

		if req.Options == nil {
			req.Options = map[string]any{}
		}

		for k, v := range m.Options {
			if _, ok := req.Options[k]; !ok {
				req.Options[k] = v
			}
		}

		// update the system prompt from the model if one isn't already specified
		if req.System == "" && m.System != "" {
			req.System = m.System
		}

		if len(m.Messages) > 0 {
			slog.Warn("embedded messages in the model not supported with '/api/generate'; try '/api/chat' instead")
		}

		fn := func(resp api.GenerateResponse) error {
			resp.Model = origModel
			resp.RemoteModel = m.Config.RemoteModel
			resp.RemoteHost = m.Config.RemoteHost

			data, err := json.Marshal(resp)
			if err != nil {
				return err
			}

			if _, err = c.Writer.Write(append(data, '\n')); err != nil {
				return err
			}
			c.Writer.Flush()
			return nil
		}

		client := api.NewClient(remoteURL, http.DefaultClient)
		err = client.Generate(c, &req, fn)
		if err != nil {
285
286
287
288
289
290
			var authError api.AuthorizationError
			if errors.As(err, &authError) {
				sURL, sErr := signinURL()
				if sErr != nil {
					slog.Error(sErr.Error())
					c.JSON(http.StatusInternalServerError, gin.H{"error": "error getting authorization details"})
291
292
					return
				}
293
294
295
296
297
298
299

				c.JSON(authError.StatusCode, gin.H{"error": "unauthorized", "signin_url": sURL})
				return
			}
			var apiError api.StatusError
			if errors.As(err, &apiError) {
				c.JSON(apiError.StatusCode, apiError)
300
301
302
303
304
305
				return
			}
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

306
307
308
309
310
311
		contentType := "application/json; charset=utf-8"
		if req.Stream != nil && *req.Stream {
			contentType = "application/x-ndjson"
		}
		c.Header("Content-Type", contentType)

312
313
314
		return
	}

Patrick Devine's avatar
Patrick Devine committed
315
	// expire the runner
Michael Yang's avatar
Michael Yang committed
316
	if req.Prompt == "" && req.KeepAlive != nil && req.KeepAlive.Duration == 0 {
317
		s.sched.expireRunner(m)
Patrick Devine's avatar
Patrick Devine committed
318
319
320
321
322
323
324
325
326
327
328

		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Response:   "",
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

329
	if req.Raw && (req.Template != "" || req.System != "" || len(req.Context) > 0) {
Michael Yang's avatar
Michael Yang committed
330
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "raw mode does not support template, system, or context"})
Michael Yang's avatar
Michael Yang committed
331
332
333
		return
	}

334
335
336
337
338
339
340
341
342
	var builtinParser parsers.Parser
	if shouldUseHarmony(m) && m.Config.Parser == "" {
		m.Config.Parser = "harmony"
	}

	if !req.Raw && m.Config.Parser != "" {
		builtinParser = parsers.ParserForName(m.Config.Parser)
		if builtinParser != nil {
			// no tools or last message for generate endpoint
Grace's avatar
Grace committed
343
			builtinParser.Init(nil, nil, req.Think)
344
		}
Michael Yang's avatar
Michael Yang committed
345
346
	}

347
348
	// Validate Think value: string values currently only allowed for harmony/gptoss models
	if req.Think != nil && req.Think.IsString() && m.Config.Parser != "harmony" {
349
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("think value %q is not supported for this model", req.Think.String())})
Michael Yang's avatar
Michael Yang committed
350
351
352
		return
	}

353
	caps := []model.Capability{model.CapabilityCompletion}
354
	if req.Suffix != "" {
355
		caps = append(caps, model.CapabilityInsert)
356
	}
357
358

	modelCaps := m.Capabilities()
359
	if slices.Contains(modelCaps, model.CapabilityThinking) {
360
		caps = append(caps, model.CapabilityThinking)
361
		if req.Think == nil {
362
363
			req.Think = &api.ThinkValue{Value: true}
		}
364
365
366
367
368
	} else {
		if req.Think != nil && req.Think.Bool() {
			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support thinking", req.Model)})
			return
		}
369
	}
370

371
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), name.String(), caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
372
373
374
375
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support generate", req.Model)})
		return
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
376
377
378
379
		handleScheduleError(c, req.Model, err)
		return
	}

380
381
	checkpointLoaded := time.Now()

382
	// load the model
Michael Yang's avatar
Michael Yang committed
383
384
385
386
387
388
389
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.GenerateResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Done:       true,
			DoneReason: "load",
		})
Michael Yang's avatar
Michael Yang committed
390
391
		return
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
392

393
394
	if slices.Contains(m.Config.ModelFamilies, "mllama") && len(req.Images) > 1 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "this model only supports one image while more than one image requested"})
395
396
397
		return
	}

Michael Yang's avatar
Michael Yang committed
398
399
	images := make([]llm.ImageData, len(req.Images))
	for i := range req.Images {
400
		images[i] = llm.ImageData{ID: i, Data: req.Images[i]}
Michael Yang's avatar
Michael Yang committed
401
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
402

Michael Yang's avatar
Michael Yang committed
403
404
	prompt := req.Prompt
	if !req.Raw {
Michael Yang's avatar
Michael Yang committed
405
		tmpl := m.Template
Michael Yang's avatar
Michael Yang committed
406
407
408
409
410
411
412
413
		if req.Template != "" {
			tmpl, err = template.Parse(req.Template)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

414
415
416
417
418
419
420
421
422
423
424
425
		var values template.Values
		if req.Suffix != "" {
			values.Prompt = prompt
			values.Suffix = req.Suffix
		} else {
			var msgs []api.Message
			if req.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: req.System})
			} else if m.System != "" {
				msgs = append(msgs, api.Message{Role: "system", Content: m.System})
			}

Michael Yang's avatar
Michael Yang committed
426
427
428
429
			if req.Context == nil {
				msgs = append(msgs, m.Messages...)
			}

430
			userMsg := api.Message{Role: "user", Content: req.Prompt}
431
			for _, i := range images {
432
				userMsg.Images = append(userMsg.Images, i.Data)
433
			}
434
			values.Messages = append(msgs, userMsg)
435
436
		}

437
		values.Think = req.Think != nil && req.Think.Bool()
Michael Yang's avatar
Michael Yang committed
438
439
		values.ThinkLevel = ""
		if req.Think != nil {
440
			values.ThinkLevel = req.Think.String()
Michael Yang's avatar
Michael Yang committed
441
		}
442
443
		values.IsThinkSet = req.Think != nil

Michael Yang's avatar
Michael Yang committed
444
445
		var b bytes.Buffer
		if req.Context != nil {
446
			slog.Warn("the context field is deprecated and will be removed in a future version of Ollama")
447
			s, err := r.Detokenize(c.Request.Context(), req.Context)
Michael Yang's avatar
Michael Yang committed
448
449
450
451
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
452
			b.WriteString(s)
Michael Yang's avatar
Michael Yang committed
453
		}
454

455
456
457
458
459
460
		// check that we're in the `api/chat`-like flow, and if so, generate the
		// prompt the same way
		// TEMP(drifkin): we should really just detect the chat-like flow and call
		// the real chat handler, but doing this as a stopgap to get renderer
		// support for generate
		if values.Messages != nil && values.Suffix == "" && req.Template == "" {
461
			prompt, images, err = chatPrompt(c.Request.Context(), m, r.Tokenize, opts, values.Messages, []api.Tool{}, req.Think, req.Truncate == nil || *req.Truncate)
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
			// TEMP(drifkin): req.Context will be removed very soon, but we're temporarily supporting it in this flow here
			if req.Context != nil {
				b.WriteString(prompt)
				prompt = b.String()
			}
		} else {
			// legacy flow
			if err := tmpl.Execute(&b, values); err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
477

478
479
			prompt = b.String()
		}
Bruce MacDonald's avatar
Bruce MacDonald committed
480
481
	}

482
483
	// If debug mode is enabled, return the rendered template instead of calling the model
	if req.DebugRenderOnly {
Devon Rifkin's avatar
Devon Rifkin committed
484
		c.JSON(http.StatusOK, api.GenerateResponse{
485
486
			Model:     req.Model,
			CreatedAt: time.Now().UTC(),
Devon Rifkin's avatar
Devon Rifkin committed
487
			DebugInfo: &api.DebugInfo{
488
489
490
491
492
493
494
				RenderedTemplate: prompt,
				ImageCount:       len(images),
			},
		})
		return
	}

495
	var thinkingState *thinking.Parser
496
	if builtinParser == nil {
Michael Yang's avatar
Michael Yang committed
497
		openingTag, closingTag := thinking.InferTags(m.Template.Template)
498
		if req.Think != nil && req.Think.Bool() && openingTag != "" && closingTag != "" {
Michael Yang's avatar
Michael Yang committed
499
500
501
502
			thinkingState = &thinking.Parser{
				OpeningTag: openingTag,
				ClosingTag: closingTag,
			}
503
504
505
			if strings.HasSuffix(strings.TrimSpace(prompt), openingTag) {
				thinkingState.AddContent(openingTag)
			}
506
507
508
		}
	}

Bruce MacDonald's avatar
Bruce MacDonald committed
509
510
	ch := make(chan any)
	go func() {
511
512
		// TODO (jmorganca): avoid building the response twice both here and below
		var sb strings.Builder
Bruce MacDonald's avatar
Bruce MacDonald committed
513
		defer close(ch)
Michael Yang's avatar
Michael Yang committed
514
		if err := r.Completion(c.Request.Context(), llm.CompletionRequest{
515
516
517
518
519
520
521
522
			Prompt:      prompt,
			Images:      images,
			Format:      req.Format,
			Options:     opts,
			Shift:       req.Shift == nil || *req.Shift,
			Truncate:    req.Truncate == nil || *req.Truncate,
			Logprobs:    req.Logprobs,
			TopLogprobs: req.TopLogprobs,
523
524
		}, func(cr llm.CompletionResponse) {
			res := api.GenerateResponse{
525
526
527
528
				Model:     req.Model,
				CreatedAt: time.Now().UTC(),
				Response:  cr.Content,
				Done:      cr.Done,
Bruce MacDonald's avatar
Bruce MacDonald committed
529
				Metrics: api.Metrics{
530
531
532
533
					PromptEvalCount:    cr.PromptEvalCount,
					PromptEvalDuration: cr.PromptEvalDuration,
					EvalCount:          cr.EvalCount,
					EvalDuration:       cr.EvalDuration,
Bruce MacDonald's avatar
Bruce MacDonald committed
534
				},
535
				Logprobs: toAPILogprobs(cr.Logprobs),
Bruce MacDonald's avatar
Bruce MacDonald committed
536
			}
537

538
539
540
541
542
543
			if builtinParser != nil {
				content, thinking, toolCalls, err := builtinParser.Add(cr.Content, cr.Done)
				if err != nil {
					ch <- gin.H{"error": err.Error()}
					return
				}
544
545
				res.Response = content
				res.Thinking = thinking
546
547
548
				if cr.Done && len(toolCalls) > 0 {
					res.ToolCalls = toolCalls
				}
549
			} else if thinkingState != nil {
Devon Rifkin's avatar
Devon Rifkin committed
550
				thinking, content := thinkingState.AddContent(cr.Content)
551
552
553
554
				res.Thinking = thinking
				res.Response = content
			}

555
556
557
558
559
			if _, err := sb.WriteString(cr.Content); err != nil {
				ch <- gin.H{"error": err.Error()}
			}

			if cr.Done {
560
561
562
563
				res.DoneReason = cr.DoneReason.String()
				res.TotalDuration = time.Since(checkpointStart)
				res.LoadDuration = checkpointLoaded.Sub(checkpointStart)

564
				if !req.Raw {
565
					tokens, err := r.Tokenize(c.Request.Context(), prompt+sb.String())
566
567
568
569
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}
570
					res.Context = tokens
571
572
573
				}
			}

574
			if builtinParser != nil {
Michael Yang's avatar
Michael Yang committed
575
576
577
578
579
580
581
582
				// only send messages with meaningful content (empty messages confuse clients)
				if res.Response != "" || res.Thinking != "" || res.Done || len(res.ToolCalls) > 0 {
					ch <- res
				}

				return
			}

583
			ch <- res
Michael Yang's avatar
Michael Yang committed
584
		}); err != nil {
585
586
587
588
589
590
			var serr api.StatusError
			if errors.As(err, &serr) {
				ch <- gin.H{"error": serr.ErrorMessage, "status": serr.StatusCode}
			} else {
				ch <- gin.H{"error": err.Error()}
			}
Bruce MacDonald's avatar
Bruce MacDonald committed
591
592
593
594
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
Michael Yang committed
595
		var r api.GenerateResponse
596
		var allLogprobs []api.Logprob
597
598
		var sbThinking strings.Builder
		var sbContent strings.Builder
Michael Yang's avatar
Michael Yang committed
599
600
		for rr := range ch {
			switch t := rr.(type) {
601
			case api.GenerateResponse:
602
603
				sbThinking.WriteString(t.Thinking)
				sbContent.WriteString(t.Response)
Michael Yang's avatar
Michael Yang committed
604
				r = t
605
606
607
608
				// Accumulate logprobs from all chunks for non-streaming response
				if len(t.Logprobs) > 0 {
					allLogprobs = append(allLogprobs, t.Logprobs...)
				}
609
610
611
612
613
614
			case gin.H:
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
				}

615
616
617
618
619
620
				status, ok := t["status"].(int)
				if !ok {
					status = http.StatusInternalServerError
				}

				c.JSON(status, gin.H{"error": msg})
Michael Yang's avatar
Michael Yang committed
621
				return
622
			default:
Michael Yang's avatar
Michael Yang committed
623
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
Bruce MacDonald's avatar
Bruce MacDonald committed
624
625
626
				return
			}
		}
627

628
629
		r.Thinking = sbThinking.String()
		r.Response = sbContent.String()
630
		r.Logprobs = allLogprobs
631

Michael Yang's avatar
Michael Yang committed
632
		c.JSON(http.StatusOK, r)
Bruce MacDonald's avatar
Bruce MacDonald committed
633
634
635
636
637
638
		return
	}

	streamResponse(c, ch)
}

639
func (s *Server) EmbedHandler(c *gin.Context) {
640
	checkpointStart := time.Now()
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
	var req api.EmbedRequest
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	truncate := true
	if req.Truncate != nil && !*req.Truncate {
		truncate = false
	}

	var input []string

	switch i := req.Input.(type) {
	case string:
		if len(i) > 0 {
			input = append(input, i)
		}
	case []any:
		for _, v := range i {
			if _, ok := v.(string); !ok {
				c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
				return
			}
			input = append(input, v.(string))
		}
	default:
673
674
675
676
		if req.Input != nil {
			c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "invalid input type"})
			return
		}
677
678
	}

679
680
681
682
683
684
	name, err := getExistingName(model.ParseName(req.Model))
	if err != nil {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		return
	}

685
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), name.String(), []model.Capability{}, req.Options, req.KeepAlive)
686
687
688
689
690
	if err != nil {
		handleScheduleError(c, req.Model, err)
		return
	}

691
692
	checkpointLoaded := time.Now()

693
694
695
696
697
	if len(input) == 0 {
		c.JSON(http.StatusOK, api.EmbedResponse{Model: req.Model, Embeddings: [][]float32{}})
		return
	}

698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
	kvData, _, err := getModelData(m.ModelPath, false)
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

	var count int
	for i, s := range input {
		tokens, err := r.Tokenize(c.Request.Context(), s)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		ctxLen := min(opts.NumCtx, int(kvData.ContextLength()))
		if len(tokens) > ctxLen {
			if !truncate {
				c.JSON(http.StatusBadRequest, gin.H{"error": "input exceeds maximum context length"})
				return
			}

			if bos := kvData.Uint("tokenizer.ggml.bos_token_id"); tokens[0] != int(bos) && kvData.Bool("add_bos_token", true) {
				ctxLen--
			}

			if eos := kvData.Uint("tokenizer.ggml.eos_token_id"); tokens[len(tokens)-1] != int(eos) && kvData.Bool("add_eos_token", true) {
				ctxLen--
			}

			slog.Info("", "ctxLen", ctxLen, "tokenCount", len(tokens))
			if ctxLen <= 0 {
				// return error if the truncated input would be empty or just special tokens
				c.JSON(http.StatusBadRequest, gin.H{"error": "input after truncation exceeds maximum context length"})
				return
			}

			tokens = tokens[:ctxLen]

			s, err = r.Detokenize(c.Request.Context(), tokens)
			if err != nil {
				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
				return
			}
		}

		count += len(tokens)

		input[i] = s
	}

748
749
750
751
	var g errgroup.Group
	embeddings := make([][]float32, len(input))
	for i, text := range input {
		g.Go(func() error {
752
			embedding, err := r.Embedding(c.Request.Context(), text)
753
754
755
			if err != nil {
				return err
			}
756
757
758
759
760
761
			// TODO: this first normalization should be done by the model
			embedding = normalize(embedding)
			if req.Dimensions > 0 && req.Dimensions < len(embedding) {
				embedding = normalize(embedding[:req.Dimensions])
			}
			embeddings[i] = embedding
762
763
			return nil
		})
764
765
	}

766
	if err := g.Wait(); err != nil {
767
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": strings.TrimSpace(err.Error())})
768
		return
769
770
771
	}

	resp := api.EmbedResponse{
772
		Model:           req.Model,
773
		Embeddings:      embeddings,
774
775
		TotalDuration:   time.Since(checkpointStart),
		LoadDuration:    checkpointLoaded.Sub(checkpointStart),
776
		PromptEvalCount: count,
777
778
779
780
781
782
783
784
785
786
	}
	c.JSON(http.StatusOK, resp)
}

func normalize(vec []float32) []float32 {
	var sum float32
	for _, v := range vec {
		sum += v * v
	}

787
	norm := float32(1.0 / max(math.Sqrt(float64(sum)), 1e-12))
788
789
790
791
792
793
	for i := range vec {
		vec[i] *= norm
	}
	return vec
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
794
func (s *Server) EmbeddingsHandler(c *gin.Context) {
Bruce MacDonald's avatar
Bruce MacDonald committed
795
	var req api.EmbeddingRequest
Michael Yang's avatar
Michael Yang committed
796
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
797
798
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
799
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
800
801
802
803
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

804
805
806
807
808
809
	name := model.ParseName(req.Model)
	if !name.IsValid() {
		c.JSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

810
	r, _, _, err := s.scheduleRunner(c.Request.Context(), name.String(), []model.Capability{}, req.Options, req.KeepAlive)
Bruce MacDonald's avatar
Bruce MacDonald committed
811
	if err != nil {
Michael Yang's avatar
Michael Yang committed
812
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
813
814
815
		return
	}

816
817
818
	// an empty request loads the model
	if req.Prompt == "" {
		c.JSON(http.StatusOK, api.EmbeddingResponse{Embedding: []float64{}})
Bruce MacDonald's avatar
Bruce MacDonald committed
819
820
821
		return
	}

822
	embedding, err := r.Embedding(c.Request.Context(), req.Prompt)
Bruce MacDonald's avatar
Bruce MacDonald committed
823
	if err != nil {
824
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": strings.TrimSpace(err.Error())})
Bruce MacDonald's avatar
Bruce MacDonald committed
825
826
827
		return
	}

828
829
830
	var e []float64
	for _, v := range embedding {
		e = append(e, float64(v))
831
832
833
	}

	resp := api.EmbeddingResponse{
834
		Embedding: e,
835
836
	}
	c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
837
838
}

839
func (s *Server) PullHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
840
	var req api.PullRequest
Michael Yang's avatar
Michael Yang committed
841
842
843
844
845
846
847
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
848
849
850
		return
	}

851
852
	name := model.ParseName(cmp.Or(req.Model, req.Name))
	if !name.IsValid() {
853
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": errtypes.InvalidModelNameErrMsg})
854
855
856
		return
	}

857
858
	name, err = getExistingName(name)
	if err != nil {
859
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
860
861
862
		return
	}

863
864
865
	ch := make(chan any)
	go func() {
		defer close(ch)
866
867
		fn := func(r api.ProgressResponse) {
			ch <- r
868
		}
869

Michael Yang's avatar
Michael Yang committed
870
		regOpts := &registryOptions{
871
872
873
			Insecure: req.Insecure,
		}

874
875
876
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

877
		if err := PullModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
878
			ch <- gin.H{"error": err.Error()}
879
880
881
		}
	}()

882
883
884
885
886
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

887
888
889
	streamResponse(c, ch)
}

890
func (s *Server) PushHandler(c *gin.Context) {
891
	var req api.PushRequest
Michael Yang's avatar
Michael Yang committed
892
893
894
895
896
897
898
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
899
900
		return
	}
Michael Yang's avatar
Michael Yang committed
901

902
	var mname string
Michael Yang's avatar
Michael Yang committed
903
	if req.Model != "" {
904
		mname = req.Model
Michael Yang's avatar
Michael Yang committed
905
	} else if req.Name != "" {
906
		mname = req.Name
Michael Yang's avatar
Michael Yang committed
907
908
	} else {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
909
910
911
		return
	}

912
913
914
	ch := make(chan any)
	go func() {
		defer close(ch)
915
916
		fn := func(r api.ProgressResponse) {
			ch <- r
917
		}
918

Michael Yang's avatar
Michael Yang committed
919
		regOpts := &registryOptions{
920
921
922
			Insecure: req.Insecure,
		}

Michael Yang's avatar
Michael Yang committed
923
924
925
		ctx, cancel := context.WithCancel(c.Request.Context())
		defer cancel()

926
927
928
929
930
931
932
		name, err := getExistingName(model.ParseName(mname))
		if err != nil {
			ch <- gin.H{"error": err.Error()}
			return
		}

		if err := PushModel(ctx, name.DisplayShortest(), regOpts, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
933
			ch <- gin.H{"error": err.Error()}
934
935
936
		}
	}()

937
938
939
940
941
	if req.Stream != nil && !*req.Stream {
		waitForStream(c, ch)
		return
	}

942
943
944
	streamResponse(c, ch)
}

945
946
947
948
// getExistingName searches the models directory for the longest prefix match of
// the input name and returns the input name with all existing parts replaced
// with each part found. If no parts are found, the input name is returned as
// is.
949
950
951
func getExistingName(n model.Name) (model.Name, error) {
	var zero model.Name
	existing, err := Manifests(true)
952
	if err != nil {
953
		return zero, err
954
	}
955
	var set model.Name // tracks parts already canonicalized
956
	for e := range existing {
957
958
959
960
961
962
963
964
965
966
967
		if set.Host == "" && strings.EqualFold(e.Host, n.Host) {
			n.Host = e.Host
		}
		if set.Namespace == "" && strings.EqualFold(e.Namespace, n.Namespace) {
			n.Namespace = e.Namespace
		}
		if set.Model == "" && strings.EqualFold(e.Model, n.Model) {
			n.Model = e.Model
		}
		if set.Tag == "" && strings.EqualFold(e.Tag, n.Tag) {
			n.Tag = e.Tag
968
969
		}
	}
970
	return n, nil
971
972
}

973
func (s *Server) DeleteHandler(c *gin.Context) {
974
975
	var r api.DeleteRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
976
977
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
978
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
979
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
980
981
982
		return
	}

983
984
985
	n := model.ParseName(cmp.Or(r.Model, r.Name))
	if !n.IsValid() {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("name %q is invalid", cmp.Or(r.Model, r.Name))})
986
987
		return
	}
Michael Yang's avatar
Michael Yang committed
988

989
990
991
992
993
994
	n, err := getExistingName(n)
	if err != nil {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", cmp.Or(r.Model, r.Name))})
		return
	}

995
	m, err := ParseNamedManifest(n)
Michael Yang's avatar
Michael Yang committed
996
	if err != nil {
997
998
999
1000
1001
1002
		switch {
		case os.IsNotExist(err):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", cmp.Or(r.Model, r.Name))})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
Michael Yang's avatar
Michael Yang committed
1003
1004
1005
		return
	}

1006
	if err := m.Remove(); err != nil {
Michael Yang's avatar
Michael Yang committed
1007
1008
1009
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
1010
1011
1012
1013
1014

	if err := m.RemoveLayers(); err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
1015
1016
}

1017
func (s *Server) ShowHandler(c *gin.Context) {
Patrick Devine's avatar
Patrick Devine committed
1018
	var req api.ShowRequest
Michael Yang's avatar
Michael Yang committed
1019
1020
1021
1022
1023
1024
1025
	err := c.ShouldBindJSON(&req)
	switch {
	case errors.Is(err, io.EOF):
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
	case err != nil:
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
1026
1027
1028
		return
	}

Michael Yang's avatar
Michael Yang committed
1029
	if req.Model != "" {
Michael Yang's avatar
Michael Yang committed
1030
		// noop
Michael Yang's avatar
Michael Yang committed
1031
	} else if req.Name != "" {
Michael Yang's avatar
Michael Yang committed
1032
		req.Model = req.Name
Michael Yang's avatar
Michael Yang committed
1033
	} else {
1034
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "model is required"})
1035
1036
1037
		return
	}

1038
	resp, err := GetModelInfo(req)
Patrick Devine's avatar
Patrick Devine committed
1039
	if err != nil {
1040
1041
		switch {
		case os.IsNotExist(err):
Michael Yang's avatar
Michael Yang committed
1042
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
1043
		case err.Error() == errtypes.InvalidModelNameErrMsg:
1044
1045
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
Patrick Devine's avatar
Patrick Devine committed
1046
1047
1048
1049
1050
1051
1052
1053
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		}
		return
	}

	c.JSON(http.StatusOK, resp)
}

1054
func GetModelInfo(req api.ShowRequest) (*api.ShowResponse, error) {
1055
1056
	name := model.ParseName(req.Model)
	if !name.IsValid() {
CYJiang's avatar
CYJiang committed
1057
		return nil, ErrModelPathInvalid
1058
1059
1060
1061
1062
1063
1064
	}
	name, err := getExistingName(name)
	if err != nil {
		return nil, err
	}

	m, err := GetModel(name.String())
Patrick Devine's avatar
Patrick Devine committed
1065
1066
1067
1068
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
1069
	modelDetails := api.ModelDetails{
1070
1071
1072
1073
1074
1075
		ParentModel:       m.ParentModel,
		Format:            m.Config.ModelFormat,
		Family:            m.Config.ModelFamily,
		Families:          m.Config.ModelFamilies,
		ParameterSize:     m.Config.ModelType,
		QuantizationLevel: m.Config.FileType,
Patrick Devine's avatar
Patrick Devine committed
1076
1077
	}

1078
	if req.System != "" {
1079
		m.System = req.System
1080
1081
	}

Michael Yang's avatar
Michael Yang committed
1082
1083
1084
	msgs := make([]api.Message, len(m.Messages))
	for i, msg := range m.Messages {
		msgs[i] = api.Message{Role: msg.Role, Content: msg.Content}
1085
1086
	}

1087
	manifest, err := ParseNamedManifest(name)
1088
1089
1090
1091
	if err != nil {
		return nil, err
	}

Patrick Devine's avatar
Patrick Devine committed
1092
	resp := &api.ShowResponse{
1093
1094
1095
1096
1097
1098
1099
		License:      strings.Join(m.License, "\n"),
		System:       m.System,
		Template:     m.Template.String(),
		Details:      modelDetails,
		Messages:     msgs,
		Capabilities: m.Capabilities(),
		ModifiedAt:   manifest.fi.ModTime(),
Patrick Devine's avatar
Patrick Devine committed
1100
1101
	}

1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
	if m.Config.RemoteHost != "" {
		resp.RemoteHost = m.Config.RemoteHost
		resp.RemoteModel = m.Config.RemoteModel

		if m.Config.ModelFamily != "" {
			resp.ModelInfo = make(map[string]any)
			resp.ModelInfo["general.architecture"] = m.Config.ModelFamily

			if m.Config.BaseName != "" {
				resp.ModelInfo["general.basename"] = m.Config.BaseName
			}

			if m.Config.ContextLen > 0 {
				resp.ModelInfo[fmt.Sprintf("%s.context_length", m.Config.ModelFamily)] = m.Config.ContextLen
			}

			if m.Config.EmbedLen > 0 {
				resp.ModelInfo[fmt.Sprintf("%s.embedding_length", m.Config.ModelFamily)] = m.Config.EmbedLen
			}
		}
	}

Patrick Devine's avatar
Patrick Devine committed
1124
1125
	var params []string
	cs := 30
1126
	for k, v := range m.Options {
Patrick Devine's avatar
Patrick Devine committed
1127
		switch val := v.(type) {
1128
		case []any:
Patrick Devine's avatar
Patrick Devine committed
1129
			for _, nv := range val {
Patrick Devine's avatar
Patrick Devine committed
1130
				params = append(params, fmt.Sprintf("%-*s %#v", cs, k, nv))
Patrick Devine's avatar
Patrick Devine committed
1131
			}
Patrick Devine's avatar
Patrick Devine committed
1132
1133
		default:
			params = append(params, fmt.Sprintf("%-*s %#v", cs, k, v))
Patrick Devine's avatar
Patrick Devine committed
1134
1135
1136
1137
		}
	}
	resp.Parameters = strings.Join(params, "\n")

Patrick Devine's avatar
Patrick Devine committed
1138
1139
1140
1141
1142
	if len(req.Options) > 0 {
		if m.Options == nil {
			m.Options = make(map[string]any)
		}
		for k, v := range req.Options {
1143
			m.Options[k] = v
1144
1145
1146
		}
	}

1147
	var sb strings.Builder
1148
	fmt.Fprintln(&sb, "# Modelfile generated by \"ollama show\"")
1149
	fmt.Fprintln(&sb, "# To build a new Modelfile based on this, replace FROM with:")
1150
1151
	fmt.Fprintf(&sb, "# FROM %s\n\n", m.ShortName)
	fmt.Fprint(&sb, m.String())
1152
	resp.Modelfile = sb.String()
1153

1154
1155
1156
1157
1158
	// skip loading tensor information if this is a remote model
	if m.Config.RemoteHost != "" && m.Config.RemoteModel != "" {
		return resp, nil
	}

1159
	kvData, tensors, err := getModelData(m.ModelPath, req.Verbose)
1160
1161
1162
	if err != nil {
		return nil, err
	}
1163

1164
1165
1166
1167
	delete(kvData, "general.name")
	delete(kvData, "tokenizer.chat_template")
	resp.ModelInfo = kvData

1168
1169
1170
1171
1172
1173
	tensorData := make([]api.Tensor, len(tensors.Items()))
	for cnt, t := range tensors.Items() {
		tensorData[cnt] = api.Tensor{Name: t.Name, Type: t.Type(), Shape: t.Shape}
	}
	resp.Tensors = tensorData

1174
	if len(m.ProjectorPaths) > 0 {
1175
		projectorData, _, err := getModelData(m.ProjectorPaths[0], req.Verbose)
1176
1177
1178
1179
1180
1181
		if err != nil {
			return nil, err
		}
		resp.ProjectorInfo = projectorData
	}

Patrick Devine's avatar
Patrick Devine committed
1182
1183
1184
	return resp, nil
}

1185
func getModelData(digest string, verbose bool) (ggml.KV, ggml.Tensors, error) {
1186
1187
1188
1189
	maxArraySize := 0
	if verbose {
		maxArraySize = -1
	}
1190
	data, err := llm.LoadModel(digest, maxArraySize)
1191
	if err != nil {
1192
		return nil, ggml.Tensors{}, err
1193
1194
	}

1195
	kv := data.KV()
1196
1197
1198
1199
1200
1201
1202
1203
1204

	if !verbose {
		for k := range kv {
			if t, ok := kv[k].([]any); len(t) > 5 && ok {
				kv[k] = []any{}
			}
		}
	}

1205
	return kv, data.Tensors(), nil
1206
1207
}

1208
func (s *Server) ListHandler(c *gin.Context) {
1209
	ms, err := Manifests(true)
Patrick Devine's avatar
Patrick Devine committed
1210
1211
1212
1213
	if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}
Michael Yang's avatar
Michael Yang committed
1214

1215
	models := []api.ListModelResponse{}
1216
1217
	for n, m := range ms {
		var cf ConfigV2
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230

		if m.Config.Digest != "" {
			f, err := m.Config.Open()
			if err != nil {
				slog.Warn("bad manifest filepath", "name", n, "error", err)
				continue
			}
			defer f.Close()

			if err := json.NewDecoder(f).Decode(&cf); err != nil {
				slog.Warn("bad manifest config", "name", n, "error", err)
				continue
			}
Patrick Devine's avatar
Patrick Devine committed
1231
		}
Michael Yang's avatar
Michael Yang committed
1232

1233
1234
		// tag should never be masked
		models = append(models, api.ListModelResponse{
1235
1236
1237
1238
1239
1240
1241
			Model:       n.DisplayShortest(),
			Name:        n.DisplayShortest(),
			RemoteModel: cf.RemoteModel,
			RemoteHost:  cf.RemoteHost,
			Size:        m.Size(),
			Digest:      m.digest,
			ModifiedAt:  m.fi.ModTime(),
1242
1243
1244
1245
1246
1247
1248
			Details: api.ModelDetails{
				Format:            cf.ModelFormat,
				Family:            cf.ModelFamily,
				Families:          cf.ModelFamilies,
				ParameterSize:     cf.ModelType,
				QuantizationLevel: cf.FileType,
			},
1249
		})
Patrick Devine's avatar
Patrick Devine committed
1250
1251
	}

1252
	slices.SortStableFunc(models, func(i, j api.ListModelResponse) int {
1253
1254
1255
1256
		// most recently modified first
		return cmp.Compare(j.ModifiedAt.Unix(), i.ModifiedAt.Unix())
	})

Michael Yang's avatar
Michael Yang committed
1257
	c.JSON(http.StatusOK, api.ListResponse{Models: models})
Patrick Devine's avatar
Patrick Devine committed
1258
1259
}

1260
func (s *Server) CopyHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
1261
1262
	var r api.CopyRequest
	if err := c.ShouldBindJSON(&r); errors.Is(err, io.EOF) {
Michael Yang's avatar
Michael Yang committed
1263
1264
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
1265
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
1266
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
1267
1268
1269
		return
	}

Michael Yang's avatar
Michael Yang committed
1270
1271
	src := model.ParseName(r.Source)
	if !src.IsValid() {
Michael Yang's avatar
Michael Yang committed
1272
1273
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("source %q is invalid", r.Source)})
		return
1274
	}
1275
1276
1277
1278
1279
	src, err := getExistingName(src)
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}
1280

Michael Yang's avatar
Michael Yang committed
1281
1282
	dst := model.ParseName(r.Destination)
	if !dst.IsValid() {
1283
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("destination %q is invalid", r.Destination)})
Patrick Devine's avatar
Patrick Devine committed
1284
1285
		return
	}
1286
1287
	dst, err = getExistingName(dst)
	if err != nil {
1288
1289
1290
1291
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

Michael Yang's avatar
Michael Yang committed
1292
1293
1294
1295
1296
	if err := CopyModel(src, dst); errors.Is(err, os.ErrNotExist) {
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found", r.Source)})
	} else if err != nil {
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
	}
Patrick Devine's avatar
Patrick Devine committed
1297
1298
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1299
func (s *Server) HeadBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	if _, err := os.Stat(path); err != nil {
		c.AbortWithStatusJSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("blob %q not found", c.Param("digest"))})
		return
	}

Michael Yang's avatar
Michael Yang committed
1311
	c.Status(http.StatusOK)
Michael Yang's avatar
Michael Yang committed
1312
1313
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1314
func (s *Server) CreateBlobHandler(c *gin.Context) {
Michael Yang's avatar
Michael Yang committed
1315
1316
	if ib, ok := intermediateBlobs[c.Param("digest")]; ok {
		p, err := GetBlobsPath(ib)
1317
1318
1319
1320
1321
1322
		if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		if _, err := os.Stat(p); errors.Is(err, os.ErrNotExist) {
Michael Yang's avatar
Michael Yang committed
1323
1324
			slog.Info("evicting intermediate blob which no longer exists", "digest", ib)
			delete(intermediateBlobs, c.Param("digest"))
1325
1326
1327
1328
1329
1330
1331
1332
1333
		} else if err != nil {
			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		} else {
			c.Status(http.StatusOK)
			return
		}
	}

1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
	path, err := GetBlobsPath(c.Param("digest"))
	if err != nil {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	_, err = os.Stat(path)
	switch {
	case errors.Is(err, os.ErrNotExist):
		// noop
	case err != nil:
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	default:
		c.Status(http.StatusOK)
		return
	}

1352
	layer, err := NewLayer(c.Request.Body, "")
Michael Yang's avatar
Michael Yang committed
1353
1354
1355
1356
1357
	if err != nil {
		c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
	}

1358
1359
	if layer.Digest != c.Param("digest") {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("digest mismatch, expected %q, got %q", c.Param("digest"), layer.Digest)})
Michael Yang's avatar
Michael Yang committed
1360
1361
1362
		return
	}

Michael Yang's avatar
Michael Yang committed
1363
	c.Status(http.StatusCreated)
Michael Yang's avatar
Michael Yang committed
1364
1365
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
func isLocalIP(ip netip.Addr) bool {
	if interfaces, err := net.Interfaces(); err == nil {
		for _, iface := range interfaces {
			addrs, err := iface.Addrs()
			if err != nil {
				continue
			}

			for _, a := range addrs {
				if parsed, _, err := net.ParseCIDR(a.String()); err == nil {
					if parsed.String() == ip.String() {
						return true
					}
				}
			}
		}
	}

	return false
}

1387
func allowedHost(host string) bool {
1388
1389
	host = strings.ToLower(host)

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1390
	if host == "" || host == "localhost" {
1391
1392
1393
		return true
	}

1394
	if hostname, err := os.Hostname(); err == nil && host == strings.ToLower(hostname) {
1395
1396
1397
		return true
	}

Michael Yang's avatar
lint  
Michael Yang committed
1398
	tlds := []string{
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1399
1400
1401
		"localhost",
		"local",
		"internal",
1402
	}
1403

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1404
	// check if the host is a local TLD
1405
1406
1407
1408
1409
1410
	for _, tld := range tlds {
		if strings.HasSuffix(host, "."+tld) {
			return true
		}
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1411
	return false
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1412
}
1413

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1414
1415
1416
func allowedHostsMiddleware(addr net.Addr) gin.HandlerFunc {
	return func(c *gin.Context) {
		if addr == nil {
1417
1418
1419
1420
			c.Next()
			return
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1421
		if addr, err := netip.ParseAddrPort(addr.String()); err == nil && !addr.Addr().IsLoopback() {
1422
1423
1424
1425
1426
1427
1428
1429
1430
			c.Next()
			return
		}

		host, _, err := net.SplitHostPort(c.Request.Host)
		if err != nil {
			host = c.Request.Host
		}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1431
		if addr, err := netip.ParseAddr(host); err == nil {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1432
			if addr.IsLoopback() || addr.IsPrivate() || addr.IsUnspecified() || isLocalIP(addr) {
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1433
1434
1435
1436
1437
				c.Next()
				return
			}
		}

1438
		if allowedHost(host) {
Michael Yang's avatar
lint  
Michael Yang committed
1439
			if c.Request.Method == http.MethodOptions {
1440
1441
1442
1443
				c.AbortWithStatus(http.StatusNoContent)
				return
			}

1444
1445
1446
1447
1448
1449
			c.Next()
			return
		}

		c.AbortWithStatus(http.StatusForbidden)
	}
1450
}
1451

1452
func (s *Server) GenerateRoutes(rc *ollama.Registry) (http.Handler, error) {
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
	corsConfig := cors.DefaultConfig()
	corsConfig.AllowWildcard = true
	corsConfig.AllowBrowserExtensions = true
	corsConfig.AllowHeaders = []string{
		"Authorization",
		"Content-Type",
		"User-Agent",
		"Accept",
		"X-Requested-With",

		// OpenAI compatibility headers
1464
1465
1466
1467
1468
		"OpenAI-Beta",
		"x-stainless-arch",
		"x-stainless-async",
		"x-stainless-custom-poll-interval",
		"x-stainless-helper-method",
1469
1470
		"x-stainless-lang",
		"x-stainless-os",
1471
1472
		"x-stainless-package-version",
		"x-stainless-poll-helper",
1473
1474
1475
1476
1477
1478
		"x-stainless-retry-count",
		"x-stainless-runtime",
		"x-stainless-runtime-version",
		"x-stainless-timeout",
	}
	corsConfig.AllowOrigins = envconfig.AllowedOrigins()
Michael Yang's avatar
Michael Yang committed
1479

Bruce MacDonald's avatar
Bruce MacDonald committed
1480
	r := gin.Default()
1481
	r.HandleMethodNotAllowed = true
1482
	r.Use(
1483
		cors.New(corsConfig),
1484
		allowedHostsMiddleware(s.addr),
1485
	)
Bruce MacDonald's avatar
Bruce MacDonald committed
1486

1487
1488
1489
1490
1491
1492
	// General
	r.HEAD("/", func(c *gin.Context) { c.String(http.StatusOK, "Ollama is running") })
	r.GET("/", func(c *gin.Context) { c.String(http.StatusOK, "Ollama is running") })
	r.HEAD("/api/version", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"version": version.Version}) })
	r.GET("/api/version", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"version": version.Version}) })

1493
	// Local model cache management (new implementation is at end of function)
1494
1495
	r.POST("/api/pull", s.PullHandler)
	r.POST("/api/push", s.PushHandler)
1496
1497
	r.HEAD("/api/tags", s.ListHandler)
	r.GET("/api/tags", s.ListHandler)
1498
	r.POST("/api/show", s.ShowHandler)
1499
	r.DELETE("/api/delete", s.DeleteHandler)
1500

1501
1502
	r.POST("/api/me", s.WhoamiHandler)

1503
1504
1505
1506
	r.POST("/api/signout", s.SignoutHandler)
	// deprecated
	r.DELETE("/api/user/keys/:encodedKey", s.SignoutHandler)

1507
1508
	// Create
	r.POST("/api/create", s.CreateHandler)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1509
1510
	r.POST("/api/blobs/:digest", s.CreateBlobHandler)
	r.HEAD("/api/blobs/:digest", s.HeadBlobHandler)
1511
1512
1513
	r.POST("/api/copy", s.CopyHandler)

	// Inference
1514
	r.GET("/api/ps", s.PsHandler)
1515
1516
1517
1518
	r.POST("/api/generate", s.GenerateHandler)
	r.POST("/api/chat", s.ChatHandler)
	r.POST("/api/embed", s.EmbedHandler)
	r.POST("/api/embeddings", s.EmbeddingsHandler)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1519

1520
	// Inference (OpenAI compatibility)
1521
1522
1523
1524
1525
	r.POST("/v1/chat/completions", middleware.ChatMiddleware(), s.ChatHandler)
	r.POST("/v1/completions", middleware.CompletionsMiddleware(), s.GenerateHandler)
	r.POST("/v1/embeddings", middleware.EmbeddingsMiddleware(), s.EmbedHandler)
	r.GET("/v1/models", middleware.ListMiddleware(), s.ListHandler)
	r.GET("/v1/models/:model", middleware.RetrieveMiddleware(), s.ShowHandler)
1526

1527
1528
1529
1530
1531
1532
	if rc != nil {
		// wrap old with new
		rs := &registry.Local{
			Client:   rc,
			Logger:   slog.Default(), // TODO(bmizerany): Take a logger, do not use slog.Default()
			Fallback: r,
1533

1534
1535
1536
			Prune: PruneLayers,
		}
		return rs, nil
1537
1538
	}

1539
	return r, nil
1540
1541
1542
}

func Serve(ln net.Listener) error {
1543
	slog.SetDefault(logutil.NewLogger(os.Stderr, envconfig.LogLevel()))
1544
	slog.Info("server config", "env", envconfig.Values())
Michael Yang's avatar
Michael Yang committed
1545

1546
1547
1548
1549
1550
1551
1552
1553
	blobsDir, err := GetBlobsPath("")
	if err != nil {
		return err
	}
	if err := fixBlobs(blobsDir); err != nil {
		return err
	}

Michael Yang's avatar
bool  
Michael Yang committed
1554
	if !envconfig.NoPrune() {
1555
1556
1557
1558
1559
1560
1561
		if _, err := Manifests(false); err != nil {
			slog.Warn("corrupt manifests detected, skipping prune operation.  Re-pull or delete to clear", "error", err)
		} else {
			// clean up unused layers and manifests
			if err := PruneLayers(); err != nil {
				return err
			}
1562

1563
1564
1565
1566
			manifestsPath, err := GetManifestPath()
			if err != nil {
				return err
			}
1567

1568
1569
1570
			if err := PruneDirectory(manifestsPath); err != nil {
				return err
			}
1571
1572
1573
		}
	}

1574
1575
	s := &Server{addr: ln.Addr()}

1576
1577
1578
1579
1580
1581
1582
	var rc *ollama.Registry
	if useClient2 {
		var err error
		rc, err = ollama.DefaultRegistry()
		if err != nil {
			return err
		}
1583
1584
	}

1585
	h, err := s.GenerateRoutes(rc)
1586
1587
1588
	if err != nil {
		return err
	}
1589

1590
1591
	http.Handle("/", h)

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1592
	ctx, done := context.WithCancel(context.Background())
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1593
1594
	schedCtx, schedDone := context.WithCancel(ctx)
	sched := InitScheduler(schedCtx)
1595
	s.sched = sched
1596

1597
	slog.Info(fmt.Sprintf("Listening on %s (version %s)", ln.Addr(), version.Version))
1598
	srvr := &http.Server{
1599
1600
1601
1602
1603
1604
1605
1606
1607
		// Use http.DefaultServeMux so we get net/http/pprof for
		// free.
		//
		// TODO(bmizerany): Decide if we want to make this
		// configurable so it is not exposed by default, or allow
		// users to bind it to a different port. This was a quick
		// and easy way to get pprof, but it may not be the best
		// way.
		Handler: nil,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1608
1609
	}

1610
1611
	// listen for a ctrl+c and stop any loaded llm
	signals := make(chan os.Signal, 1)
1612
	signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM)
1613
1614
	go func() {
		<-signals
1615
		srvr.Close()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1616
		schedDone()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1617
		sched.unloadAllRunners()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1618
		done()
1619
1620
	}()

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1621
	s.sched.Run(schedCtx)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1622

1623
1624
1625
1626
	// register the experimental webp decoder
	// so webp images can be used in multimodal inputs
	image.RegisterFormat("webp", "RIFF????WEBP", webp.Decode, webp.DecodeConfig)

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1627
1628
	// At startup we retrieve GPU information so we can get log messages before loading a model
	// This will log warnings to the log in case we have problems with detected GPUs
1629
1630
	gpus := discover.GPUDevices(ctx, nil)
	discover.LogDetails(gpus)
1631

1632
1633
1634
1635
1636
1637
1638
1639
1640
	var totalVRAM uint64
	for _, gpu := range gpus {
		totalVRAM += gpu.TotalMemory - envconfig.GpuOverhead()
	}
	if totalVRAM < lowVRAMThreshold {
		s.lowVRAM = true
		slog.Info("entering low vram mode", "total vram", format.HumanBytes2(totalVRAM), "threshold", format.HumanBytes2(lowVRAMThreshold))
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1641
1642
1643
1644
1645
1646
1647
	err = srvr.Serve(ln)
	// If server is closed from the signal handler, wait for the ctx to be done
	// otherwise error out quickly
	if !errors.Is(err, http.ErrServerClosed) {
		return err
	}
	<-ctx.Done()
1648
	return nil
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1649
}
Michael Yang's avatar
Michael Yang committed
1650

1651
func waitForStream(c *gin.Context, ch chan any) {
1652
	c.Header("Content-Type", "application/json")
1653
	var latest api.ProgressResponse
1654
1655
1656
	for resp := range ch {
		switch r := resp.(type) {
		case api.ProgressResponse:
1657
			latest = r
1658
		case gin.H:
Josh's avatar
Josh committed
1659
1660
1661
1662
			status, ok := r["status"].(int)
			if !ok {
				status = http.StatusInternalServerError
			}
1663
1664
1665
			errorMsg, ok := r["error"].(string)
			if !ok {
				errorMsg = "unknown error"
1666
			}
1667
1668
			c.JSON(status, gin.H{"error": errorMsg})
			return
1669
		default:
1670
			c.JSON(http.StatusInternalServerError, gin.H{"error": "unknown message type"})
1671
1672
1673
			return
		}
	}
1674
1675

	c.JSON(http.StatusOK, latest)
1676
1677
}

Michael Yang's avatar
Michael Yang committed
1678
func streamResponse(c *gin.Context, ch chan any) {
1679
	c.Header("Content-Type", "application/x-ndjson")
Michael Yang's avatar
Michael Yang committed
1680
1681
1682
1683
1684
1685
	c.Stream(func(w io.Writer) bool {
		val, ok := <-ch
		if !ok {
			return false
		}

1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
		// errors are provided as a gin.H with an "error" field and
		// an optional "status" field.  For errors that are streamed
		// before any content, we need to set the status code and
		// content type for the error.
		if h, ok := val.(gin.H); ok {
			if e, ok := h["error"].(string); ok {
				status, ok := h["status"].(int)
				if !ok {
					status = http.StatusInternalServerError
				}

				if !c.Writer.Written() {
					c.Header("Content-Type", "application/json")
					c.JSON(status, gin.H{"error": e})
				} else {
					if err := json.NewEncoder(c.Writer).Encode(gin.H{"error": e}); err != nil {
						slog.Error("streamResponse failed to encode json error", "error", err)
					}
				}

				return false
			}
		}

Michael Yang's avatar
Michael Yang committed
1710
1711
		bts, err := json.Marshal(val)
		if err != nil {
1712
			slog.Info(fmt.Sprintf("streamResponse: json.Marshal failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1713
1714
1715
			return false
		}

1716
		// Delineate chunks with new-line delimiter
Michael Yang's avatar
Michael Yang committed
1717
1718
		bts = append(bts, '\n')
		if _, err := w.Write(bts); err != nil {
1719
			slog.Info(fmt.Sprintf("streamResponse: w.Write failed with %s", err))
Michael Yang's avatar
Michael Yang committed
1720
1721
1722
1723
1724
1725
			return false
		}

		return true
	})
}
Bruce MacDonald's avatar
Bruce MacDonald committed
1726

1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
func (s *Server) WhoamiHandler(c *gin.Context) {
	// todo allow other hosts
	u, err := url.Parse("https://ollama.com")
	if err != nil {
		slog.Error(err.Error())
		c.JSON(http.StatusInternalServerError, gin.H{"error": "URL parse error"})
		return
	}

	client := api.NewClient(u, http.DefaultClient)
	user, err := client.Whoami(c)
	if err != nil {
		slog.Error(err.Error())
	}
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754

	// user isn't signed in
	if user != nil && user.Name == "" {
		sURL, sErr := signinURL()
		if sErr != nil {
			slog.Error(sErr.Error())
			c.JSON(http.StatusInternalServerError, gin.H{"error": "error getting authorization details"})
			return
		}

		c.JSON(http.StatusUnauthorized, gin.H{"error": "unauthorized", "signin_url": sURL})
		return
	}

1755
1756
1757
1758
	c.JSON(http.StatusOK, user)
}

func (s *Server) SignoutHandler(c *gin.Context) {
1759
1760
1761
1762
1763
1764
1765
1766
	pubKey, err := auth.GetPublicKey()
	if err != nil {
		slog.Error("couldn't get public key", "error", err)
		c.JSON(http.StatusInternalServerError, gin.H{"error": "there was an error signing out"})
		return
	}

	encKey := base64.RawURLEncoding.EncodeToString([]byte(pubKey))
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776

	// todo allow other hosts
	u, err := url.Parse("https://ollama.com")
	if err != nil {
		slog.Error(err.Error())
		c.JSON(http.StatusInternalServerError, gin.H{"error": "URL parse error"})
		return
	}

	client := api.NewClient(u, http.DefaultClient)
1777
	err = client.Disconnect(c, encKey)
1778
	if err != nil {
1779
1780
1781
		var authError api.AuthorizationError
		if errors.As(err, &authError) {
			c.JSON(http.StatusUnauthorized, gin.H{"error": "you are not currently signed in"})
1782
1783
1784
1785
1786
1787
1788
1789
1790
			return
		}
		c.JSON(http.StatusInternalServerError, gin.H{"error": "there was an error signing out"})
		return
	}

	c.JSON(http.StatusOK, nil)
}

1791
func (s *Server) PsHandler(c *gin.Context) {
1792
	models := []api.ProcessModelResponse{}
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803

	for _, v := range s.sched.loaded {
		model := v.model
		modelDetails := api.ModelDetails{
			Format:            model.Config.ModelFormat,
			Family:            model.Config.ModelFamily,
			Families:          model.Config.ModelFamilies,
			ParameterSize:     model.Config.ModelType,
			QuantizationLevel: model.Config.FileType,
		}

1804
		mr := api.ProcessModelResponse{
1805
1806
			Model:     model.ShortName,
			Name:      model.ShortName,
Jesse Gross's avatar
Jesse Gross committed
1807
1808
			Size:      int64(v.totalSize),
			SizeVRAM:  int64(v.vramSize),
1809
1810
1811
1812
			Digest:    model.Digest,
			Details:   modelDetails,
			ExpiresAt: v.expiresAt,
		}
1813
		if v.Options != nil {
Jesse Gross's avatar
Jesse Gross committed
1814
			mr.ContextLength = v.Options.NumCtx
1815
		}
1816
1817
1818
1819
1820
1821
1822
1823
		// The scheduler waits to set expiresAt, so if a model is loading it's
		// possible that it will be set to the unix epoch. For those cases, just
		// calculate the time w/ the sessionDuration instead.
		var epoch time.Time
		if v.expiresAt == epoch {
			mr.ExpiresAt = time.Now().Add(v.sessionDuration)
		}

1824
1825
1826
		models = append(models, mr)
	}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1827
1828
1829
1830
1831
	slices.SortStableFunc(models, func(i, j api.ProcessModelResponse) int {
		// longest duration remaining listed first
		return cmp.Compare(j.ExpiresAt.Unix(), i.ExpiresAt.Unix())
	})

1832
	c.JSON(http.StatusOK, api.ProcessResponse{Models: models})
1833
1834
}

Grace's avatar
Grace committed
1835
1836
1837
1838
1839
1840
1841
1842
1843
func toolCallId() string {
	const letterBytes = "abcdefghijklmnopqrstuvwxyz0123456789"
	b := make([]byte, 8)
	for i := range b {
		b[i] = letterBytes[rand.Intn(len(letterBytes))]
	}
	return "call_" + strings.ToLower(string(b))
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1844
func (s *Server) ChatHandler(c *gin.Context) {
1845
1846
	checkpointStart := time.Now()

Bruce MacDonald's avatar
Bruce MacDonald committed
1847
	var req api.ChatRequest
Michael Yang's avatar
Michael Yang committed
1848
	if err := c.ShouldBindJSON(&req); errors.Is(err, io.EOF) {
Bruce MacDonald's avatar
Bruce MacDonald committed
1849
1850
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "missing request body"})
		return
Michael Yang's avatar
Michael Yang committed
1851
	} else if err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
1852
1853
1854
1855
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

1856
1857
1858
1859
1860
	if req.TopLogprobs < 0 || req.TopLogprobs > 20 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "top_logprobs must be between 0 and 20"})
		return
	}

1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
	name := model.ParseName(req.Model)
	if !name.IsValid() {
		c.JSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

	name, err := getExistingName(name)
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": "model is required"})
		return
	}

	m, err := GetModel(req.Model)
	if err != nil {
		switch {
		case os.IsNotExist(err):
			c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model '%s' not found", req.Model)})
		case err.Error() == errtypes.InvalidModelNameErrMsg:
			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		default:
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
Patrick Devine's avatar
Patrick Devine committed
1882
		}
1883
1884
1885
		return
	}

1886
1887
1888
1889
1890
	if req.TopLogprobs < 0 || req.TopLogprobs > 20 {
		c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "top_logprobs must be between 0 and 20"})
		return
	}

1891
	// expire the runner
Michael Yang's avatar
Michael Yang committed
1892
	if len(req.Messages) == 0 && req.KeepAlive != nil && req.KeepAlive.Duration == 0 {
1893
		s.sched.expireRunner(m)
Patrick Devine's avatar
Patrick Devine committed
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904

		c.JSON(http.StatusOK, api.ChatResponse{
			Model:      req.Model,
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
			Done:       true,
			DoneReason: "unload",
		})
		return
	}

1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
	if m.Config.RemoteHost != "" && m.Config.RemoteModel != "" {
		origModel := req.Model

		remoteURL, err := url.Parse(m.Config.RemoteHost)
		if err != nil {
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

		if !slices.Contains(envconfig.Remotes(), remoteURL.Hostname()) {
			slog.Info("remote model", "remotes", envconfig.Remotes(), "remoteURL", m.Config.RemoteHost, "hostname", remoteURL.Hostname())
			c.JSON(http.StatusBadRequest, gin.H{"error": "this server cannot run this remote model"})
			return
		}

		req.Model = m.Config.RemoteModel
		if req.Options == nil {
			req.Options = map[string]any{}
		}

1925
1926
1927
1928
1929
1930
		var msgs []api.Message
		if len(req.Messages) > 0 {
			msgs = append(m.Messages, req.Messages...)
			if req.Messages[0].Role != "system" && m.System != "" {
				msgs = append([]api.Message{{Role: "system", Content: m.System}}, msgs...)
			}
1931
		}
1932

1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
		msgs = filterThinkTags(msgs, m)
		req.Messages = msgs

		for k, v := range m.Options {
			if _, ok := req.Options[k]; !ok {
				req.Options[k] = v
			}
		}

		fn := func(resp api.ChatResponse) error {
			resp.Model = origModel
			resp.RemoteModel = m.Config.RemoteModel
			resp.RemoteHost = m.Config.RemoteHost

			data, err := json.Marshal(resp)
			if err != nil {
				return err
			}

			if _, err = c.Writer.Write(append(data, '\n')); err != nil {
				return err
			}
			c.Writer.Flush()
			return nil
		}

		client := api.NewClient(remoteURL, http.DefaultClient)
		err = client.Chat(c, &req, fn)
		if err != nil {
1962
1963
1964
1965
1966
1967
			var authError api.AuthorizationError
			if errors.As(err, &authError) {
				sURL, sErr := signinURL()
				if sErr != nil {
					slog.Error(sErr.Error())
					c.JSON(http.StatusInternalServerError, gin.H{"error": "error getting authorization details"})
1968
1969
					return
				}
1970
1971
1972
1973
1974
1975
1976

				c.JSON(authError.StatusCode, gin.H{"error": "unauthorized", "signin_url": sURL})
				return
			}
			var apiError api.StatusError
			if errors.As(err, &apiError) {
				c.JSON(apiError.StatusCode, apiError)
1977
1978
				return
			}
1979
1980
1981
1982
			c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
			return
		}

1983
1984
1985
1986
1987
1988
		contentType := "application/json; charset=utf-8"
		if req.Stream != nil && *req.Stream {
			contentType = "application/x-ndjson"
		}
		c.Header("Content-Type", contentType)

1989
1990
1991
		return
	}

1992
	caps := []model.Capability{model.CapabilityCompletion}
1993
	if len(req.Tools) > 0 {
1994
		caps = append(caps, model.CapabilityTools)
Michael Yang's avatar
tools  
Michael Yang committed
1995
	}
1996
1997

	modelCaps := m.Capabilities()
1998
	if slices.Contains(modelCaps, model.CapabilityThinking) {
1999
		caps = append(caps, model.CapabilityThinking)
2000
		if req.Think == nil {
2001
2002
			req.Think = &api.ThinkValue{Value: true}
		}
2003
2004
2005
2006
2007
	} else {
		if req.Think != nil && req.Think.Bool() {
			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support thinking", req.Model)})
			return
		}
2008
	}
Michael Yang's avatar
tools  
Michael Yang committed
2009

2010
	r, m, opts, err := s.scheduleRunner(c.Request.Context(), name.String(), caps, req.Options, req.KeepAlive)
Michael Yang's avatar
Michael Yang committed
2011
2012
	if errors.Is(err, errCapabilityCompletion) {
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%q does not support chat", req.Model)})
Bruce MacDonald's avatar
Bruce MacDonald committed
2013
		return
Michael Yang's avatar
Michael Yang committed
2014
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
2015
		handleScheduleError(c, req.Model, err)
Bruce MacDonald's avatar
Bruce MacDonald committed
2016
2017
		return
	}
Michael Yang's avatar
Michael Yang committed
2018

2019
2020
	checkpointLoaded := time.Now()

Michael Yang's avatar
Michael Yang committed
2021
2022
	if len(req.Messages) == 0 {
		c.JSON(http.StatusOK, api.ChatResponse{
2023
			Model:      req.Model,
Michael Yang's avatar
Michael Yang committed
2024
2025
			CreatedAt:  time.Now().UTC(),
			Message:    api.Message{Role: "assistant"},
2026
2027
			Done:       true,
			DoneReason: "load",
Michael Yang's avatar
Michael Yang committed
2028
		})
2029
2030
2031
		return
	}

Michael Yang's avatar
Michael Yang committed
2032
	msgs := append(m.Messages, req.Messages...)
2033
	if req.Messages[0].Role != "system" && m.System != "" {
Michael Yang's avatar
Michael Yang committed
2034
		msgs = append([]api.Message{{Role: "system", Content: m.System}}, msgs...)
2035
	}
2036
	msgs = filterThinkTags(msgs, m)
2037

2038
2039
	if shouldUseHarmony(m) && m.Config.Parser == "" {
		m.Config.Parser = "harmony"
Devon Rifkin's avatar
Devon Rifkin committed
2040
2041
	}

2042
	var builtinParser parsers.Parser
2043
	processedTools := req.Tools
2044

2045
2046
2047
2048
2049
2050
2051
2052
2053
	if m.Config.Parser != "" {
		builtinParser = parsers.ParserForName(m.Config.Parser)
		if builtinParser != nil {
			// Determine last message for chat prefill
			var lastMessage *api.Message
			if len(msgs) > 0 {
				lastMessage = &msgs[len(msgs)-1]
			}
			// Initialize parser and get processed tools
Grace's avatar
Grace committed
2054
			processedTools = builtinParser.Init(req.Tools, lastMessage, req.Think)
2055
2056
2057
		}
	}

2058
2059
	truncate := req.Truncate == nil || *req.Truncate
	prompt, images, err := chatPrompt(c.Request.Context(), m, r.Tokenize, opts, msgs, processedTools, req.Think, truncate)
Michael Yang's avatar
Michael Yang committed
2060
	if err != nil {
2061
		slog.Error("chat prompt error", "error", err)
Michael Yang's avatar
Michael Yang committed
2062
2063
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
		return
2064
2065
	}

2066
2067
	// If debug mode is enabled, return the rendered template instead of calling the model
	if req.DebugRenderOnly {
Devon Rifkin's avatar
Devon Rifkin committed
2068
		c.JSON(http.StatusOK, api.ChatResponse{
2069
2070
			Model:     req.Model,
			CreatedAt: time.Now().UTC(),
Devon Rifkin's avatar
Devon Rifkin committed
2071
			DebugInfo: &api.DebugInfo{
2072
2073
2074
2075
2076
2077
2078
				RenderedTemplate: prompt,
				ImageCount:       len(images),
			},
		})
		return
	}

2079
2080
	// Validate Think value: string values currently only allowed for harmony/gptoss models
	if req.Think != nil && req.Think.IsString() && m.Config.Parser != "harmony" {
2081
		c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("think value %q is not supported for this model", req.Think.String())})
Michael Yang's avatar
Michael Yang committed
2082
2083
2084
		return
	}

2085
2086
	var thinkingState *thinking.Parser
	openingTag, closingTag := thinking.InferTags(m.Template.Template)
2087
	if req.Think != nil && req.Think.Bool() && openingTag != "" && closingTag != "" {
2088
		thinkingState = &thinking.Parser{
Devon Rifkin's avatar
Devon Rifkin committed
2089
2090
			OpeningTag: openingTag,
			ClosingTag: closingTag,
2091
		}
2092
2093
2094
2095

		if strings.HasSuffix(strings.TrimSpace(prompt), openingTag) {
			thinkingState.AddContent(openingTag)
		}
2096
2097
	}

2098
	var toolParser *tools.Parser
2099
	if len(req.Tools) > 0 && (builtinParser == nil || !builtinParser.HasToolSupport()) {
2100
		toolParser = tools.NewParser(m.Template.Template, req.Tools)
2101
2102
	}

2103
2104
2105
2106
2107
2108
2109
	type structuredOutputsState int
	const (
		structuredOutputsState_None structuredOutputsState = iota
		structuredOutputsState_ReadyToApply
		structuredOutputsState_Applying
	)

Bruce MacDonald's avatar
Bruce MacDonald committed
2110
2111
2112
	ch := make(chan any)
	go func() {
		defer close(ch)
2113

2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
		structuredOutputsState := structuredOutputsState_None

		for {
			var tb strings.Builder

			currentFormat := req.Format
			// structured outputs via double request is enabled when:
			// 1. the model supports the thinking capability and
			// 2. it uses a built-in parser or our generic thinking parser

			// Note that the current approach does not work for (potential future)
			// non-thinking models that emit anything before actual content. This
			// current approach uses the transition from parsed thinking content to
			// parsed non-thinking content as the signal to turn constraining on

			if req.Format != nil && structuredOutputsState == structuredOutputsState_None && ((builtinParser != nil || thinkingState != nil) && slices.Contains(m.Capabilities(), model.CapabilityThinking)) {
				currentFormat = nil
Michael Yang's avatar
Michael Yang committed
2131
2132
			}

2133
2134
2135
			// sets up new context given parent context per request
			ctx, cancel := context.WithCancel(c.Request.Context())
			err := r.Completion(ctx, llm.CompletionRequest{
2136
2137
2138
2139
2140
2141
2142
2143
				Prompt:      prompt,
				Images:      images,
				Format:      currentFormat,
				Options:     opts,
				Shift:       req.Shift == nil || *req.Shift,
				Truncate:    truncate,
				Logprobs:    req.Logprobs,
				TopLogprobs: req.TopLogprobs,
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
			}, func(r llm.CompletionResponse) {
				res := api.ChatResponse{
					Model:     req.Model,
					CreatedAt: time.Now().UTC(),
					Message:   api.Message{Role: "assistant", Content: r.Content},
					Done:      r.Done,
					Metrics: api.Metrics{
						PromptEvalCount:    r.PromptEvalCount,
						PromptEvalDuration: r.PromptEvalDuration,
						EvalCount:          r.EvalCount,
						EvalDuration:       r.EvalDuration,
					},
2156
					Logprobs: toAPILogprobs(r.Logprobs),
2157
				}
2158

2159
2160
2161
2162
2163
				if r.Done {
					res.DoneReason = r.DoneReason.String()
					res.TotalDuration = time.Since(checkpointStart)
					res.LoadDuration = checkpointLoaded.Sub(checkpointStart)
				}
Devon Rifkin's avatar
Devon Rifkin committed
2164

2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
				if builtinParser != nil {
					slog.Log(context.TODO(), logutil.LevelTrace, "builtin parser input", "parser", m.Config.Parser, "content", r.Content)

					content, thinking, toolCalls, err := builtinParser.Add(r.Content, r.Done)
					if err != nil {
						ch <- gin.H{"error": err.Error()}
						return
					}

					res.Message.Content = content
					res.Message.Thinking = thinking
Grace's avatar
Grace committed
2176
2177
2178
					for i := range toolCalls {
						toolCalls[i].ID = toolCallId()
					}
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
					res.Message.ToolCalls = toolCalls

					tb.WriteString(thinking)
					// we are now receiving content from the model - we should start applying structured outputs
					if structuredOutputsState == structuredOutputsState_None && req.Format != nil && tb.String() != "" && res.Message.Content != "" {
						structuredOutputsState = structuredOutputsState_ReadyToApply
						cancel()
						return
					}

					if res.Message.Content != "" || res.Message.Thinking != "" || len(res.Message.ToolCalls) > 0 || r.Done {
						slog.Log(context.TODO(), logutil.LevelTrace, "builtin parser output", "parser", m.Config.Parser, "content", content, "thinking", thinking, "toolCalls", toolCalls, "done", r.Done)
						ch <- res
					} else {
						slog.Log(context.TODO(), logutil.LevelTrace, "builtin parser empty output", "parser", m.Config.Parser)
					}
Devon Rifkin's avatar
Devon Rifkin committed
2195
2196
2197
					return
				}

2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
				if thinkingState != nil {
					thinkingContent, remainingContent := thinkingState.AddContent(res.Message.Content)
					if thinkingContent == "" && remainingContent == "" && !r.Done {
						// need to accumulate more to decide what to send
						return
					}
					res.Message.Thinking = thinkingContent
					tb.WriteString(thinkingContent)
					// emit the collected thinking text before restarting with structured outputs and clear unstructured content
					// to avoid leaking mixed tokens like "</think>Hello"
					if structuredOutputsState == structuredOutputsState_None && req.Format != nil && tb.String() != "" && remainingContent != "" {
						structuredOutputsState = structuredOutputsState_ReadyToApply
						res.Message.Content = ""
						ch <- res
						cancel()
						return
					}
					res.Message.Content = remainingContent
Devon Rifkin's avatar
Devon Rifkin committed
2216
2217
				}

2218
2219
2220
2221
2222
				if len(req.Tools) > 0 {
					toolCalls, content := toolParser.Add(res.Message.Content)
					if len(content) > 0 {
						res.Message.Content = content
					} else if len(toolCalls) > 0 {
Grace's avatar
Grace committed
2223
2224
2225
						for i := range toolCalls {
							toolCalls[i].ID = toolCallId()
						}
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
						res.Message.ToolCalls = toolCalls
						res.Message.Content = ""
					} else if res.Message.Thinking != "" {
						// don't return
					} else {
						if r.Done {
							res.Message.Content = toolParser.Content()
							ch <- res
						}
						return
					}
				}
2238

2239
2240
2241
2242
2243
2244
				ch <- res
			})
			if err != nil {
				if structuredOutputsState == structuredOutputsState_ReadyToApply && strings.Contains(err.Error(), "context canceled") && c.Request.Context().Err() == nil {
					// only ignores error if it's a context cancellation due to setting structured outputs
				} else {
2245
2246
2247
2248
2249
2250
					var serr api.StatusError
					if errors.As(err, &serr) {
						ch <- gin.H{"error": serr.ErrorMessage, "status": serr.StatusCode}
					} else {
						ch <- gin.H{"error": err.Error()}
					}
2251
2252
2253
2254
					return
				}
			}

2255
2256
2257
2258
2259
2260
2261
2262
2263
			// ignored structured outputs cancellation falls through to here, start a new request with the structured outputs and updated prompt. use the
			if structuredOutputsState == structuredOutputsState_ReadyToApply {
				structuredOutputsState = structuredOutputsState_Applying
				msg := api.Message{
					Role:     "assistant",
					Thinking: tb.String(),
				}

				msgs = append(msgs, msg)
2264
				prompt, _, err = chatPrompt(c.Request.Context(), m, r.Tokenize, opts, msgs, processedTools, req.Think, truncate)
2265
2266
2267
				if err != nil {
					slog.Error("chat prompt error applying structured outputs", "error", err)
					ch <- gin.H{"error": err.Error()}
2268
					return
2269
				}
2270
2271
2272
2273
2274
2275
2276
2277
				// force constraining by terminating thinking header, the parser is already at this state
				// when the last message is thinking, the rendered for gpt-oss cannot disambiguate between having the
				// model continue thinking or ending thinking and outputting the final message.
				// TODO(parthsareen): consider adding prefill disambiguation logic to the renderer for structured outputs.
				if shouldUseHarmony(m) || (builtinParser != nil && m.Config.Parser == "harmony") {
					prompt += "<|end|><|start|>assistant<|channel|>final<|message|>"
				}
				continue
2278
			}
2279

2280
			break
Bruce MacDonald's avatar
Bruce MacDonald committed
2281
2282
2283
2284
		}
	}()

	if req.Stream != nil && !*req.Stream {
Michael Yang's avatar
tools  
Michael Yang committed
2285
		var resp api.ChatResponse
2286
		var toolCalls []api.ToolCall
2287
		var allLogprobs []api.Logprob
2288
2289
		var sbThinking strings.Builder
		var sbContent strings.Builder
Michael Yang's avatar
Michael Yang committed
2290
2291
		for rr := range ch {
			switch t := rr.(type) {
2292
			case api.ChatResponse:
2293
2294
				sbThinking.WriteString(t.Message.Thinking)
				sbContent.WriteString(t.Message.Content)
Michael Yang's avatar
tools  
Michael Yang committed
2295
				resp = t
2296
2297
2298
				if len(req.Tools) > 0 {
					toolCalls = append(toolCalls, t.Message.ToolCalls...)
				}
2299
2300
2301
2302
				// Accumulate logprobs from all chunks for non-streaming response
				if len(t.Logprobs) > 0 {
					allLogprobs = append(allLogprobs, t.Logprobs...)
				}
2303
2304
2305
2306
2307
2308
			case gin.H:
				msg, ok := t["error"].(string)
				if !ok {
					msg = "unexpected error format in response"
				}

2309
2310
2311
2312
2313
2314
				status, ok := t["status"].(int)
				if !ok {
					status = http.StatusInternalServerError
				}

				c.JSON(status, gin.H{"error": msg})
Michael Yang's avatar
Michael Yang committed
2315
				return
2316
			default:
Michael Yang's avatar
Michael Yang committed
2317
				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected response"})
2318
				return
Bruce MacDonald's avatar
Bruce MacDonald committed
2319
2320
			}
		}
2321

2322
2323
		resp.Message.Content = sbContent.String()
		resp.Message.Thinking = sbThinking.String()
2324
		resp.Logprobs = allLogprobs
2325

2326
2327
		if len(toolCalls) > 0 {
			resp.Message.ToolCalls = toolCalls
Michael Yang's avatar
tools  
Michael Yang committed
2328
2329
2330
		}

		c.JSON(http.StatusOK, resp)
Bruce MacDonald's avatar
Bruce MacDonald committed
2331
2332
2333
2334
2335
		return
	}

	streamResponse(c, ch)
}
2336

Michael Yang's avatar
Michael Yang committed
2337
func handleScheduleError(c *gin.Context, name string, err error) {
Michael Yang's avatar
Michael Yang committed
2338
	switch {
2339
	case errors.Is(err, errCapabilities), errors.Is(err, errRequired):
Michael Yang's avatar
Michael Yang committed
2340
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
2341
	case errors.Is(err, context.Canceled):
2342
		c.JSON(499, gin.H{"error": "request canceled"})
Michael Yang's avatar
Michael Yang committed
2343
	case errors.Is(err, ErrMaxQueue):
2344
		c.JSON(http.StatusServiceUnavailable, gin.H{"error": err.Error()})
Michael Yang's avatar
Michael Yang committed
2345
2346
	case errors.Is(err, os.ErrNotExist):
		c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("model %q not found, try pulling it first", name)})
Michael Yang's avatar
Michael Yang committed
2347
2348
	default:
		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
2349
2350
	}
}
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362

func filterThinkTags(msgs []api.Message, m *Model) []api.Message {
	if m.Config.ModelFamily == "qwen3" || model.ParseName(m.Name).Model == "deepseek-r1" {
		finalUserIndex := -1
		for i, msg := range msgs {
			if msg.Role == "user" {
				finalUserIndex = i
			}
		}

		for i, msg := range msgs {
			if msg.Role == "assistant" && i < finalUserIndex {
2363
2364
2365
2366
2367
				// TODO(drifkin): this is from before we added proper thinking support.
				// However, even if thinking is not enabled (and therefore we shouldn't
				// change the user output), we should probably perform this filtering
				// for all thinking models (not just qwen3 & deepseek-r1) since it tends
				// to save tokens and improve quality.
2368
				thinkingState := &thinking.Parser{
Devon Rifkin's avatar
Devon Rifkin committed
2369
2370
					OpeningTag: "<think>",
					ClosingTag: "</think>",
2371
				}
Devon Rifkin's avatar
Devon Rifkin committed
2372
				_, content := thinkingState.AddContent(msg.Content)
2373
				msgs[i].Content = content
2374
2375
2376
2377
2378
			}
		}
	}
	return msgs
}