cmd.go 31.7 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package cmd

import (
Michael Yang's avatar
Michael Yang committed
4
	"bufio"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
5
	"context"
6
7
	"crypto/ed25519"
	"crypto/rand"
8
	"encoding/json"
9
	"encoding/pem"
Michael Yang's avatar
Michael Yang committed
10
	"errors"
Bruce MacDonald's avatar
Bruce MacDonald committed
11
	"fmt"
Michael Yang's avatar
Michael Yang committed
12
	"io"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
13
	"log"
14
	"math"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
15
	"net"
16
	"net/http"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
17
	"os"
18
	"os/signal"
19
	"path/filepath"
20
	"runtime"
21
	"sort"
Michael Yang's avatar
Michael Yang committed
22
	"strconv"
Michael Yang's avatar
Michael Yang committed
23
	"strings"
24
	"sync/atomic"
25
	"syscall"
Michael Yang's avatar
Michael Yang committed
26
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
27

28
	"github.com/containerd/console"
29
	"github.com/mattn/go-runewidth"
Patrick Devine's avatar
Patrick Devine committed
30
	"github.com/olekukonko/tablewriter"
Michael Yang's avatar
Michael Yang committed
31
	"github.com/spf13/cobra"
32
	"golang.org/x/crypto/ssh"
33
	"golang.org/x/term"
Michael Yang's avatar
Michael Yang committed
34

35
	"github.com/ollama/ollama/api"
36
	"github.com/ollama/ollama/envconfig"
37
	"github.com/ollama/ollama/format"
38
	"github.com/ollama/ollama/parser"
39
	"github.com/ollama/ollama/progress"
Jesse Gross's avatar
Jesse Gross committed
40
	"github.com/ollama/ollama/runner"
41
	"github.com/ollama/ollama/server"
42
	"github.com/ollama/ollama/types/model"
43
	"github.com/ollama/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
44
45
)

46
var errModelfileNotFound = errors.New("specified Modelfile wasn't found")
47
48

func getModelfileName(cmd *cobra.Command) (string, error) {
49
	filename, _ := cmd.Flags().GetString("file")
50
51
52
53
54
55

	if filename == "" {
		filename = "Modelfile"
	}

	absName, err := filepath.Abs(filename)
56
	if err != nil {
57
		return "", err
58
59
	}

60
	_, err = os.Stat(absName)
61
	if err != nil {
62
		return "", err
63
	}
64

65
66
67
68
	return absName, nil
}

func CreateHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
69
70
71
	p := progress.NewProgress(os.Stderr)
	defer p.Stop()

72
73
74
75
76
77
78
79
80
81
	var reader io.Reader

	filename, err := getModelfileName(cmd)
	if os.IsNotExist(err) {
		if filename == "" {
			reader = strings.NewReader("FROM .\n")
		} else {
			return errModelfileNotFound
		}
	} else if err != nil {
Michael Yang's avatar
Michael Yang committed
82
		return err
83
84
85
86
87
88
89
90
	} else {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}

		reader = f
		defer f.Close()
Michael Yang's avatar
Michael Yang committed
91
92
	}

93
	modelfile, err := parser.ParseFile(reader)
Michael Yang's avatar
Michael Yang committed
94
95
96
97
	if err != nil {
		return err
	}

98
99
100
101
	status := "gathering model components"
	spinner := progress.NewSpinner(status)
	p.Add(status, spinner)

102
	req, err := modelfile.CreateRequest(filepath.Dir(filename))
Michael Yang's avatar
Michael Yang committed
103
104
105
	if err != nil {
		return err
	}
106
	spinner.Stop()
Michael Yang's avatar
Michael Yang committed
107

108
109
110
111
112
	req.Name = args[0]
	quantize, _ := cmd.Flags().GetString("quantize")
	if quantize != "" {
		req.Quantize = quantize
	}
113

114
115
116
117
118
	client, err := api.ClientFromEnvironment()
	if err != nil {
		return err
	}

119
120
121
122
	if len(req.Files) > 0 {
		fileMap := map[string]string{}
		for f, digest := range req.Files {
			if _, err := createBlob(cmd, client, f, digest, p); err != nil {
Michael Yang's avatar
Michael Yang committed
123
124
				return err
			}
125
126
127
128
			fileMap[filepath.Base(f)] = digest
		}
		req.Files = fileMap
	}
Michael Yang's avatar
Michael Yang committed
129

130
131
132
133
	if len(req.Adapters) > 0 {
		fileMap := map[string]string{}
		for f, digest := range req.Adapters {
			if _, err := createBlob(cmd, client, f, digest, p); err != nil {
Michael Yang's avatar
Michael Yang committed
134
135
				return err
			}
136
			fileMap[filepath.Base(f)] = digest
Michael Yang's avatar
Michael Yang committed
137
		}
138
		req.Adapters = fileMap
Michael Yang's avatar
Michael Yang committed
139
	}
Michael Yang's avatar
Michael Yang committed
140

Michael Yang's avatar
Michael Yang committed
141
	bars := make(map[string]*progress.Bar)
142
	fn := func(resp api.ProgressResponse) error {
Michael Yang's avatar
Michael Yang committed
143
144
145
		if resp.Digest != "" {
			bar, ok := bars[resp.Digest]
			if !ok {
146
				bar = progress.NewBar(fmt.Sprintf("pulling %s...", resp.Digest[7:19]), resp.Total, resp.Completed)
Michael Yang's avatar
Michael Yang committed
147
148
149
150
151
152
153
154
155
156
157
158
159
				bars[resp.Digest] = bar
				p.Add(resp.Digest, bar)
			}

			bar.Set(resp.Completed)
		} else if status != resp.Status {
			spinner.Stop()

			status = resp.Status
			spinner = progress.NewSpinner(status)
			p.Add(status, spinner)
		}

160
161
162
		return nil
	}

163
	if err := client.Create(cmd.Context(), req, fn); err != nil {
164
165
166
		if strings.Contains(err.Error(), "path or Modelfile are required") {
			return fmt.Errorf("the ollama server must be updated to use `ollama create` with this client")
		}
167
168
169
170
171
172
		return err
	}

	return nil
}

173
174
func createBlob(cmd *cobra.Command, client *api.Client, path string, digest string, p *progress.Progress) (string, error) {
	realPath, err := filepath.EvalSymlinks(path)
Michael Yang's avatar
Michael Yang committed
175
176
177
178
	if err != nil {
		return "", err
	}

179
	bin, err := os.Open(realPath)
180
181
182
183
184
	if err != nil {
		return "", err
	}
	defer bin.Close()

185
186
187
188
189
190
191
192
	// Get file info to retrieve the size
	fileInfo, err := bin.Stat()
	if err != nil {
		return "", err
	}
	fileSize := fileInfo.Size()

	var pw progressWriter
193
194
195
196
	status := fmt.Sprintf("copying file %s 0%%", digest)
	spinner := progress.NewSpinner(status)
	p.Add(status, spinner)
	defer spinner.Stop()
197
198
199
200
201
202
203
204
205
206

	done := make(chan struct{})
	defer close(done)

	go func() {
		ticker := time.NewTicker(60 * time.Millisecond)
		defer ticker.Stop()
		for {
			select {
			case <-ticker.C:
207
				spinner.SetMessage(fmt.Sprintf("copying file %s %d%%", digest, int(100*pw.n.Load()/fileSize)))
208
			case <-done:
209
				spinner.SetMessage(fmt.Sprintf("copying file %s 100%%", digest))
210
211
212
213
214
215
				return
			}
		}
	}()

	if err = client.CreateBlob(cmd.Context(), digest, io.TeeReader(bin, &pw)); err != nil {
216
217
218
219
220
		return "", err
	}
	return digest, nil
}

221
222
223
224
225
226
227
228
229
type progressWriter struct {
	n atomic.Int64
}

func (w *progressWriter) Write(p []byte) (n int, err error) {
	w.n.Add(int64(len(p)))
	return len(p), nil
}

Patrick Devine's avatar
Patrick Devine committed
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
func loadOrUnloadModel(cmd *cobra.Command, opts *runOptions) error {
	p := progress.NewProgress(os.Stderr)
	defer p.StopAndClear()

	spinner := progress.NewSpinner("")
	p.Add("", spinner)

	client, err := api.ClientFromEnvironment()
	if err != nil {
		return err
	}

	req := &api.GenerateRequest{
		Model:     opts.Model,
		KeepAlive: opts.KeepAlive,
	}

	return client.Generate(cmd.Context(), req, func(api.GenerateResponse) error { return nil })
}

func StopHandler(cmd *cobra.Command, args []string) error {
	opts := &runOptions{
		Model:     args[0],
		KeepAlive: &api.Duration{Duration: 0},
	}
	if err := loadOrUnloadModel(cmd, opts); err != nil {
		if strings.Contains(err.Error(), "not found") {
			return fmt.Errorf("couldn't find model \"%s\" to stop", args[0])
		}
259
		return err
Patrick Devine's avatar
Patrick Devine committed
260
261
262
263
	}
	return nil
}

264
func RunHandler(cmd *cobra.Command, args []string) error {
265
266
267
	interactive := true

	opts := runOptions{
268
269
270
		Model:    args[0],
		WordWrap: os.Getenv("TERM") == "xterm-256color",
		Options:  map[string]interface{}{},
271
272
273
274
275
276
277
278
	}

	format, err := cmd.Flags().GetString("format")
	if err != nil {
		return err
	}
	opts.Format = format

279
280
281
282
283
284
285
286
287
288
289
290
	keepAlive, err := cmd.Flags().GetString("keepalive")
	if err != nil {
		return err
	}
	if keepAlive != "" {
		d, err := time.ParseDuration(keepAlive)
		if err != nil {
			return err
		}
		opts.KeepAlive = &api.Duration{Duration: d}
	}

291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
	prompts := args[1:]
	// prepend stdin to the prompt if provided
	if !term.IsTerminal(int(os.Stdin.Fd())) {
		in, err := io.ReadAll(os.Stdin)
		if err != nil {
			return err
		}

		prompts = append([]string{string(in)}, prompts...)
		opts.WordWrap = false
		interactive = false
	}
	opts.Prompt = strings.Join(prompts, " ")
	if len(prompts) > 0 {
		interactive = false
	}
307
308
309
310
	// Be quiet if we're redirecting to a pipe or file
	if !term.IsTerminal(int(os.Stdout.Fd())) {
		interactive = false
	}
311
312
313
314
315
316
317

	nowrap, err := cmd.Flags().GetBool("nowordwrap")
	if err != nil {
		return err
	}
	opts.WordWrap = !nowrap

318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
	// Fill out the rest of the options based on information about the
	// model.
	client, err := api.ClientFromEnvironment()
	if err != nil {
		return err
	}

	name := args[0]
	info, err := func() (*api.ShowResponse, error) {
		showReq := &api.ShowRequest{Name: name}
		info, err := client.Show(cmd.Context(), showReq)
		var se api.StatusError
		if errors.As(err, &se) && se.StatusCode == http.StatusNotFound {
			if err := PullHandler(cmd, []string{name}); err != nil {
				return nil, err
			}
			return client.Show(cmd.Context(), &api.ShowRequest{Name: name})
		}
		return info, err
	}()
	if err != nil {
		return err
340
341
	}

342
343
344
345
346
347
348
349
350
351
	if len(info.ProjectorInfo) != 0 {
		opts.MultiModal = true
	}
	for k := range info.ModelInfo {
		if strings.Contains(k, ".vision.") {
			opts.MultiModal = true
			break
		}
	}

352
353
354
	opts.ParentModel = info.Details.ParentModel

	if interactive {
Patrick Devine's avatar
Patrick Devine committed
355
		if err := loadOrUnloadModel(cmd, &opts); err != nil {
Michael Yang's avatar
Michael Yang committed
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
			return err
		}

		for _, msg := range info.Messages {
			switch msg.Role {
			case "user":
				fmt.Printf(">>> %s\n", msg.Content)
			case "assistant":
				state := &displayResponseState{}
				displayResponse(msg.Content, opts.WordWrap, state)
				fmt.Println()
				fmt.Println()
			}
		}

371
372
373
		return generateInteractive(cmd, opts)
	}
	return generate(cmd, opts)
Bruce MacDonald's avatar
Bruce MacDonald committed
374
375
}

376
func PushHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
377
	client, err := api.ClientFromEnvironment()
378
379
380
	if err != nil {
		return err
	}
381

382
383
384
385
386
	insecure, err := cmd.Flags().GetBool("insecure")
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
387
388
389
390
	p := progress.NewProgress(os.Stderr)
	defer p.Stop()

	bars := make(map[string]*progress.Bar)
391
392
	var status string
	var spinner *progress.Spinner
Michael Yang's avatar
Michael Yang committed
393

394
	fn := func(resp api.ProgressResponse) error {
Michael Yang's avatar
Michael Yang committed
395
		if resp.Digest != "" {
396
397
398
			if spinner != nil {
				spinner.Stop()
			}
Michael Yang's avatar
Michael Yang committed
399
400
401

			bar, ok := bars[resp.Digest]
			if !ok {
402
				bar = progress.NewBar(fmt.Sprintf("pushing %s...", resp.Digest[7:19]), resp.Total, resp.Completed)
Michael Yang's avatar
Michael Yang committed
403
404
405
406
407
408
				bars[resp.Digest] = bar
				p.Add(resp.Digest, bar)
			}

			bar.Set(resp.Completed)
		} else if status != resp.Status {
409
410
411
			if spinner != nil {
				spinner.Stop()
			}
Michael Yang's avatar
Michael Yang committed
412
413
414
415
416
417

			status = resp.Status
			spinner = progress.NewSpinner(status)
			p.Add(status, spinner)
		}

418
419
420
		return nil
	}

Michael Yang's avatar
Michael Yang committed
421
	request := api.PushRequest{Name: args[0], Insecure: insecure}
422
423

	n := model.ParseName(args[0])
Michael Yang's avatar
Michael Yang committed
424
	if err := client.Push(cmd.Context(), &request, fn); err != nil {
425
426
427
428
429
430
		if spinner != nil {
			spinner.Stop()
		}
		if strings.Contains(err.Error(), "access denied") {
			return errors.New("you are not authorized to push to this namespace, create the model under a namespace you own")
		}
Michael Yang's avatar
Michael Yang committed
431
432
433
		return err
	}

434
	p.Stop()
435
	spinner.Stop()
436
437
438
439
440
441
442
443

	destination := n.String()
	if strings.HasSuffix(n.Host, ".ollama.ai") || strings.HasSuffix(n.Host, ".ollama.com") {
		destination = "https://ollama.com/" + strings.TrimSuffix(n.DisplayShortest(), ":latest")
	}
	fmt.Printf("\nYou can find your model at:\n\n")
	fmt.Printf("\t%s\n", destination)

Michael Yang's avatar
Michael Yang committed
444
	return nil
445
446
}

447
func ListHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
448
	client, err := api.ClientFromEnvironment()
449
450
451
	if err != nil {
		return err
	}
Patrick Devine's avatar
Patrick Devine committed
452

Michael Yang's avatar
Michael Yang committed
453
	models, err := client.List(cmd.Context())
Patrick Devine's avatar
Patrick Devine committed
454
455
456
457
458
459
460
	if err != nil {
		return err
	}

	var data [][]string

	for _, m := range models.Models {
461
		if len(args) == 0 || strings.HasPrefix(strings.ToLower(m.Name), strings.ToLower(args[0])) {
462
			data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), format.HumanTime(m.ModifiedAt, "Never")})
Michael Yang's avatar
Michael Yang committed
463
		}
Patrick Devine's avatar
Patrick Devine committed
464
465
466
	}

	table := tablewriter.NewWriter(os.Stdout)
Patrick Devine's avatar
Patrick Devine committed
467
	table.SetHeader([]string{"NAME", "ID", "SIZE", "MODIFIED"})
Patrick Devine's avatar
Patrick Devine committed
468
469
470
471
472
	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeaderLine(false)
	table.SetBorder(false)
	table.SetNoWhiteSpace(true)
Michael Yang's avatar
Michael Yang committed
473
	table.SetTablePadding("    ")
Patrick Devine's avatar
Patrick Devine committed
474
475
476
477
478
479
	table.AppendBulk(data)
	table.Render()

	return nil
}

480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
func ListRunningHandler(cmd *cobra.Command, args []string) error {
	client, err := api.ClientFromEnvironment()
	if err != nil {
		return err
	}

	models, err := client.ListRunning(cmd.Context())
	if err != nil {
		return err
	}

	var data [][]string

	for _, m := range models.Models {
		if len(args) == 0 || strings.HasPrefix(m.Name, args[0]) {
			var procStr string
			switch {
			case m.SizeVRAM == 0:
				procStr = "100% CPU"
			case m.SizeVRAM == m.Size:
				procStr = "100% GPU"
			case m.SizeVRAM > m.Size || m.Size == 0:
				procStr = "Unknown"
			default:
				sizeCPU := m.Size - m.SizeVRAM
				cpuPercent := math.Round(float64(sizeCPU) / float64(m.Size) * 100)
				procStr = fmt.Sprintf("%d%%/%d%% CPU/GPU", int(cpuPercent), int(100-cpuPercent))
			}
Patrick Devine's avatar
Patrick Devine committed
508
509
510
511
512
513
514
515
516

			var until string
			delta := time.Since(m.ExpiresAt)
			if delta > 0 {
				until = "Stopping..."
			} else {
				until = format.HumanTime(m.ExpiresAt, "Never")
			}
			data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), procStr, until})
517
518
519
520
521
522
523
524
525
526
		}
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"NAME", "ID", "SIZE", "PROCESSOR", "UNTIL"})
	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeaderLine(false)
	table.SetBorder(false)
	table.SetNoWhiteSpace(true)
Michael Yang's avatar
Michael Yang committed
527
	table.SetTablePadding("    ")
528
529
530
531
532
533
	table.AppendBulk(data)
	table.Render()

	return nil
}

534
func DeleteHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
535
	client, err := api.ClientFromEnvironment()
536
537
538
	if err != nil {
		return err
	}
539

540
541
542
543
544
545
546
547
548
549
550
	// Unload the model if it's running before deletion
	opts := &runOptions{
		Model:     args[0],
		KeepAlive: &api.Duration{Duration: 0},
	}
	if err := loadOrUnloadModel(cmd, opts); err != nil {
		if !strings.Contains(err.Error(), "not found") {
			return fmt.Errorf("unable to stop existing running model \"%s\": %s", args[0], err)
		}
	}

551
552
	for _, name := range args {
		req := api.DeleteRequest{Name: name}
Michael Yang's avatar
Michael Yang committed
553
		if err := client.Delete(cmd.Context(), &req); err != nil {
554
555
556
			return err
		}
		fmt.Printf("deleted '%s'\n", name)
557
558
559
560
	}
	return nil
}

Patrick Devine's avatar
Patrick Devine committed
561
func ShowHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
562
	client, err := api.ClientFromEnvironment()
Patrick Devine's avatar
Patrick Devine committed
563
564
565
566
567
568
569
570
571
	if err != nil {
		return err
	}

	license, errLicense := cmd.Flags().GetBool("license")
	modelfile, errModelfile := cmd.Flags().GetBool("modelfile")
	parameters, errParams := cmd.Flags().GetBool("parameters")
	system, errSystem := cmd.Flags().GetBool("system")
	template, errTemplate := cmd.Flags().GetBool("template")
572
	verbose, errVerbose := cmd.Flags().GetBool("verbose")
Patrick Devine's avatar
Patrick Devine committed
573

574
	for _, boolErr := range []error{errLicense, errModelfile, errParams, errSystem, errTemplate, errVerbose} {
Patrick Devine's avatar
Patrick Devine committed
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
		if boolErr != nil {
			return errors.New("error retrieving flags")
		}
	}

	flagsSet := 0
	showType := ""

	if license {
		flagsSet++
		showType = "license"
	}

	if modelfile {
		flagsSet++
		showType = "modelfile"
	}

	if parameters {
		flagsSet++
		showType = "parameters"
	}

	if system {
		flagsSet++
		showType = "system"
	}

	if template {
		flagsSet++
		showType = "template"
	}

	if flagsSet > 1 {
609
		return errors.New("only one of '--license', '--modelfile', '--parameters', '--system', or '--template' can be specified")
610
611
	}

612
	req := api.ShowRequest{Name: args[0], Verbose: verbose}
613
614
615
616
	resp, err := client.Show(cmd.Context(), &req)
	if err != nil {
		return err
	}
617

618
	if flagsSet == 1 {
619
620
621
622
623
624
625
626
		switch showType {
		case "license":
			fmt.Println(resp.License)
		case "modelfile":
			fmt.Println(resp.Modelfile)
		case "parameters":
			fmt.Println(resp.Parameters)
		case "system":
627
			fmt.Print(resp.System)
628
		case "template":
629
			fmt.Print(resp.Template)
630
631
632
		}

		return nil
Patrick Devine's avatar
Patrick Devine committed
633
634
	}

635
	return showInfo(resp, verbose, os.Stdout)
636
637
}

638
func showInfo(resp *api.ShowResponse, verbose bool, w io.Writer) error {
Michael Yang's avatar
Michael Yang committed
639
640
641
642
643
644
645
	tableRender := func(header string, rows func() [][]string) {
		fmt.Fprintln(w, " ", header)
		table := tablewriter.NewWriter(w)
		table.SetAlignment(tablewriter.ALIGN_LEFT)
		table.SetBorder(false)
		table.SetNoWhiteSpace(true)
		table.SetTablePadding("    ")
646

Michael Yang's avatar
Michael Yang committed
647
648
649
		switch header {
		case "Template", "System", "License":
			table.SetColWidth(100)
650
651
		}

Michael Yang's avatar
Michael Yang committed
652
653
654
		table.AppendBulk(rows())
		table.Render()
		fmt.Fprintln(w)
Patrick Devine's avatar
Patrick Devine committed
655
656
	}

Michael Yang's avatar
Michael Yang committed
657
658
659
660
661
662
663
664
665
666
667
668
669
670
	tableRender("Model", func() (rows [][]string) {
		if resp.ModelInfo != nil {
			arch := resp.ModelInfo["general.architecture"].(string)
			rows = append(rows, []string{"", "architecture", arch})
			rows = append(rows, []string{"", "parameters", format.HumanNumber(uint64(resp.ModelInfo["general.parameter_count"].(float64)))})
			rows = append(rows, []string{"", "context length", strconv.FormatFloat(resp.ModelInfo[fmt.Sprintf("%s.context_length", arch)].(float64), 'f', -1, 64)})
			rows = append(rows, []string{"", "embedding length", strconv.FormatFloat(resp.ModelInfo[fmt.Sprintf("%s.embedding_length", arch)].(float64), 'f', -1, 64)})
		} else {
			rows = append(rows, []string{"", "architecture", resp.Details.Family})
			rows = append(rows, []string{"", "parameters", resp.Details.ParameterSize})
		}
		rows = append(rows, []string{"", "quantization", resp.Details.QuantizationLevel})
		return
	})
671

Michael Yang's avatar
Michael Yang committed
672
673
674
675
676
677
678
679
680
	if resp.ProjectorInfo != nil {
		tableRender("Projector", func() (rows [][]string) {
			arch := resp.ProjectorInfo["general.architecture"].(string)
			rows = append(rows, []string{"", "architecture", arch})
			rows = append(rows, []string{"", "parameters", format.HumanNumber(uint64(resp.ProjectorInfo["general.parameter_count"].(float64)))})
			rows = append(rows, []string{"", "embedding length", strconv.FormatFloat(resp.ProjectorInfo[fmt.Sprintf("%s.vision.embedding_length", arch)].(float64), 'f', -1, 64)})
			rows = append(rows, []string{"", "dimensions", strconv.FormatFloat(resp.ProjectorInfo[fmt.Sprintf("%s.vision.projection_dim", arch)].(float64), 'f', -1, 64)})
			return
		})
681
682
	}

Michael Yang's avatar
Michael Yang committed
683
684
685
686
687
688
689
690
691
692
	if resp.Parameters != "" {
		tableRender("Parameters", func() (rows [][]string) {
			scanner := bufio.NewScanner(strings.NewReader(resp.Parameters))
			for scanner.Scan() {
				if text := scanner.Text(); text != "" {
					rows = append(rows, append([]string{""}, strings.Fields(text)...))
				}
			}
			return
		})
693
694
	}

695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
	if resp.ModelInfo != nil && verbose {
		tableRender("Metadata", func() (rows [][]string) {
			keys := make([]string, 0, len(resp.ModelInfo))
			for k := range resp.ModelInfo {
				keys = append(keys, k)
			}
			sort.Strings(keys)

			for _, k := range keys {
				var v string
				switch vData := resp.ModelInfo[k].(type) {
				case string:
					v = vData
				case float64:
					v = fmt.Sprintf("%g", vData)
				case []any:
					n := 3
					if len(vData) < n {
						n = len(vData)
					}
					v = fmt.Sprintf("%v", vData[:n])
				default:
					v = fmt.Sprintf("%T", vData)
				}
				rows = append(rows, []string{"", k, v})
			}
			return
		})
	}

	if len(resp.Tensors) > 0 && verbose {
		tableRender("Tensors", func() (rows [][]string) {
			for _, t := range resp.Tensors {
				rows = append(rows, []string{"", t.Name, t.Type, fmt.Sprint(t.Shape)})
			}
			return
		})
	}

Michael Yang's avatar
Michael Yang committed
734
735
736
737
738
	head := func(s string, n int) (rows [][]string) {
		scanner := bufio.NewScanner(strings.NewReader(s))
		for scanner.Scan() && (len(rows) < n || n < 0) {
			if text := scanner.Text(); text != "" {
				rows = append(rows, []string{"", strings.TrimSpace(text)})
739
740
			}
		}
Michael Yang's avatar
Michael Yang committed
741
		return
742
743
	}

Michael Yang's avatar
Michael Yang committed
744
745
746
747
748
	if resp.System != "" {
		tableRender("System", func() [][]string {
			return head(resp.System, 2)
		})
	}
749

Michael Yang's avatar
Michael Yang committed
750
751
752
753
	if resp.License != "" {
		tableRender("License", func() [][]string {
			return head(resp.License, 2)
		})
754
	}
Michael Yang's avatar
Michael Yang committed
755
756

	return nil
757
758
}

Patrick Devine's avatar
Patrick Devine committed
759
func CopyHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
760
	client, err := api.ClientFromEnvironment()
761
762
763
	if err != nil {
		return err
	}
Patrick Devine's avatar
Patrick Devine committed
764
765

	req := api.CopyRequest{Source: args[0], Destination: args[1]}
Michael Yang's avatar
Michael Yang committed
766
	if err := client.Copy(cmd.Context(), &req); err != nil {
Patrick Devine's avatar
Patrick Devine committed
767
768
769
770
771
772
		return err
	}
	fmt.Printf("copied '%s' to '%s'\n", args[0], args[1])
	return nil
}

773
func PullHandler(cmd *cobra.Command, args []string) error {
774
775
776
777
778
	insecure, err := cmd.Flags().GetBool("insecure")
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
779
	client, err := api.ClientFromEnvironment()
780
781
782
	if err != nil {
		return err
	}
783

Michael Yang's avatar
Michael Yang committed
784
785
786
787
788
	p := progress.NewProgress(os.Stderr)
	defer p.Stop()

	bars := make(map[string]*progress.Bar)

789
790
	var status string
	var spinner *progress.Spinner
Michael Yang's avatar
Michael Yang committed
791

792
	fn := func(resp api.ProgressResponse) error {
Michael Yang's avatar
Michael Yang committed
793
		if resp.Digest != "" {
794
795
796
			if spinner != nil {
				spinner.Stop()
			}
Michael Yang's avatar
Michael Yang committed
797
798
799

			bar, ok := bars[resp.Digest]
			if !ok {
800
				bar = progress.NewBar(fmt.Sprintf("pulling %s...", resp.Digest[7:19]), resp.Total, resp.Completed)
Michael Yang's avatar
Michael Yang committed
801
802
803
804
805
806
				bars[resp.Digest] = bar
				p.Add(resp.Digest, bar)
			}

			bar.Set(resp.Completed)
		} else if status != resp.Status {
807
808
809
			if spinner != nil {
				spinner.Stop()
			}
Michael Yang's avatar
Michael Yang committed
810
811
812
813
814
815

			status = resp.Status
			spinner = progress.NewSpinner(status)
			p.Add(status, spinner)
		}

816
817
		return nil
	}
818

Michael Yang's avatar
Michael Yang committed
819
	request := api.PullRequest{Name: args[0], Insecure: insecure}
Michael Yang's avatar
Michael Yang committed
820
	if err := client.Pull(cmd.Context(), &request, fn); err != nil {
Michael Yang's avatar
Michael Yang committed
821
822
823
824
		return err
	}

	return nil
Michael Yang's avatar
Michael Yang committed
825
826
}

827
828
type generateContextKey string

829
type runOptions struct {
830
831
832
833
834
835
836
837
838
839
	Model       string
	ParentModel string
	Prompt      string
	Messages    []api.Message
	WordWrap    bool
	Format      string
	System      string
	Images      []api.ImageData
	Options     map[string]interface{}
	MultiModal  bool
840
	KeepAlive   *api.Duration
841
842
}

843
844
845
846
847
848
849
850
851
type displayResponseState struct {
	lineLength int
	wordBuffer string
}

func displayResponse(content string, wordWrap bool, state *displayResponseState) {
	termWidth, _, _ := term.GetSize(int(os.Stdout.Fd()))
	if wordWrap && termWidth >= 10 {
		for _, ch := range content {
Josh Yan's avatar
Josh Yan committed
852
853
			if state.lineLength+1 > termWidth-5 {
				if runewidth.StringWidth(state.wordBuffer) > termWidth-10 {
854
855
856
857
858
859
860
					fmt.Printf("%s%c", state.wordBuffer, ch)
					state.wordBuffer = ""
					state.lineLength = 0
					continue
				}

				// backtrack the length of the last word and clear to the end of the line
861
862
				a := runewidth.StringWidth(state.wordBuffer)
				if a > 0 {
863
					fmt.Printf("\x1b[%dD", a)
864
865
				}
				fmt.Printf("\x1b[K\n")
866
				fmt.Printf("%s%c", state.wordBuffer, ch)
867
868
869
				chWidth := runewidth.RuneWidth(ch)

				state.lineLength = runewidth.StringWidth(state.wordBuffer) + chWidth
870
871
			} else {
				fmt.Print(string(ch))
872
873
874
875
				state.lineLength += runewidth.RuneWidth(ch)
				if runewidth.RuneWidth(ch) >= 2 {
					state.wordBuffer = ""
					continue
Josh Yan's avatar
Josh Yan committed
876
				}
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937

				switch ch {
				case ' ':
					state.wordBuffer = ""
				case '\n':
					state.lineLength = 0
				default:
					state.wordBuffer += string(ch)
				}
			}
		}
	} else {
		fmt.Printf("%s%s", state.wordBuffer, content)
		if len(state.wordBuffer) > 0 {
			state.wordBuffer = ""
		}
	}
}

func chat(cmd *cobra.Command, opts runOptions) (*api.Message, error) {
	client, err := api.ClientFromEnvironment()
	if err != nil {
		return nil, err
	}

	p := progress.NewProgress(os.Stderr)
	defer p.StopAndClear()

	spinner := progress.NewSpinner("")
	p.Add("", spinner)

	cancelCtx, cancel := context.WithCancel(cmd.Context())
	defer cancel()

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT)

	go func() {
		<-sigChan
		cancel()
	}()

	var state *displayResponseState = &displayResponseState{}
	var latest api.ChatResponse
	var fullResponse strings.Builder
	var role string

	fn := func(response api.ChatResponse) error {
		p.StopAndClear()

		latest = response

		role = response.Message.Role
		content := response.Message.Content
		fullResponse.WriteString(content)

		displayResponse(content, opts.WordWrap, state)

		return nil
	}

938
939
940
941
	if opts.Format == "json" {
		opts.Format = `"` + opts.Format + `"`
	}

942
943
944
	req := &api.ChatRequest{
		Model:    opts.Model,
		Messages: opts.Messages,
945
		Format:   json.RawMessage(opts.Format),
946
947
948
		Options:  opts.Options,
	}

949
950
951
952
	if opts.KeepAlive != nil {
		req.KeepAlive = opts.KeepAlive
	}

953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
	if err := client.Chat(cancelCtx, req, fn); err != nil {
		if errors.Is(err, context.Canceled) {
			return nil, nil
		}
		return nil, err
	}

	if len(opts.Messages) > 0 {
		fmt.Println()
		fmt.Println()
	}

	verbose, err := cmd.Flags().GetBool("verbose")
	if err != nil {
		return nil, err
	}

	if verbose {
		latest.Summary()
	}

	return &api.Message{Role: role, Content: fullResponse.String()}, nil
}

func generate(cmd *cobra.Command, opts runOptions) error {
Michael Yang's avatar
Michael Yang committed
978
	client, err := api.ClientFromEnvironment()
Patrick Devine's avatar
Patrick Devine committed
979
	if err != nil {
980
		return err
Patrick Devine's avatar
Patrick Devine committed
981
	}
Michael Yang's avatar
Michael Yang committed
982

Michael Yang's avatar
Michael Yang committed
983
	p := progress.NewProgress(os.Stderr)
984
	defer p.StopAndClear()
985

Michael Yang's avatar
Michael Yang committed
986
987
988
	spinner := progress.NewSpinner("")
	p.Add("", spinner)

989
990
991
992
993
994
995
	var latest api.GenerateResponse

	generateContext, ok := cmd.Context().Value(generateContextKey("context")).([]int)
	if !ok {
		generateContext = []int{}
	}

Michael Yang's avatar
Michael Yang committed
996
	ctx, cancel := context.WithCancel(cmd.Context())
997
998
999
1000
1001
1002
1003
1004
1005
1006
	defer cancel()

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT)

	go func() {
		<-sigChan
		cancel()
	}()

1007
	var state *displayResponseState = &displayResponseState{}
1008

1009
	fn := func(response api.GenerateResponse) error {
Michael Yang's avatar
Michael Yang committed
1010
		p.StopAndClear()
1011

Patrick Devine's avatar
Patrick Devine committed
1012
		latest = response
1013
		content := response.Response
1014

1015
		displayResponse(content, opts.WordWrap, state)
1016

Patrick Devine's avatar
Patrick Devine committed
1017
1018
		return nil
	}
1019

1020
1021
1022
1023
1024
1025
1026
	if opts.MultiModal {
		opts.Prompt, opts.Images, err = extractFileData(opts.Prompt)
		if err != nil {
			return err
		}
	}

1027
1028
1029
1030
	if opts.Format == "json" {
		opts.Format = `"` + opts.Format + `"`
	}

Michael Yang's avatar
Michael Yang committed
1031
	request := api.GenerateRequest{
1032
1033
1034
1035
		Model:     opts.Model,
		Prompt:    opts.Prompt,
		Context:   generateContext,
		Images:    opts.Images,
1036
		Format:    json.RawMessage(opts.Format),
1037
1038
1039
		System:    opts.System,
		Options:   opts.Options,
		KeepAlive: opts.KeepAlive,
Michael Yang's avatar
Michael Yang committed
1040
1041
1042
	}

	if err := client.Generate(ctx, &request, fn); err != nil {
1043
		if errors.Is(err, context.Canceled) {
1044
			return nil
1045
		}
1046
		return err
Patrick Devine's avatar
Patrick Devine committed
1047
	}
1048

1049
	if opts.Prompt != "" {
Michael Yang's avatar
Michael Yang committed
1050
1051
		fmt.Println()
		fmt.Println()
Patrick Devine's avatar
Patrick Devine committed
1052
	}
1053

1054
1055
1056
1057
	if !latest.Done {
		return nil
	}

Patrick Devine's avatar
Patrick Devine committed
1058
1059
	verbose, err := cmd.Flags().GetBool("verbose")
	if err != nil {
1060
		return err
Patrick Devine's avatar
Patrick Devine committed
1061
	}
Michael Yang's avatar
Michael Yang committed
1062

Patrick Devine's avatar
Patrick Devine committed
1063
1064
	if verbose {
		latest.Summary()
Michael Yang's avatar
Michael Yang committed
1065
	}
Michael Yang's avatar
Michael Yang committed
1066

Patrick Devine's avatar
Patrick Devine committed
1067
1068
1069
	ctx = context.WithValue(cmd.Context(), generateContextKey("context"), latest.Context)
	cmd.SetContext(ctx)

1070
	return nil
Michael Yang's avatar
Michael Yang committed
1071
1072
}

1073
func RunServer(_ *cobra.Command, _ []string) error {
Michael Yang's avatar
Michael Yang committed
1074
	if err := initializeKeypair(); err != nil {
1075
1076
1077
		return err
	}

Michael Yang's avatar
host  
Michael Yang committed
1078
	ln, err := net.Listen("tcp", envconfig.Host().Host)
1079
1080
1081
	if err != nil {
		return err
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1082

1083
1084
1085
1086
1087
1088
	err = server.Serve(ln)
	if errors.Is(err, http.ErrServerClosed) {
		return nil
	}

	return err
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1089
1090
}

1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
func initializeKeypair() error {
	home, err := os.UserHomeDir()
	if err != nil {
		return err
	}

	privKeyPath := filepath.Join(home, ".ollama", "id_ed25519")
	pubKeyPath := filepath.Join(home, ".ollama", "id_ed25519.pub")

	_, err = os.Stat(privKeyPath)
	if os.IsNotExist(err) {
		fmt.Printf("Couldn't find '%s'. Generating new private key.\n", privKeyPath)
Michael Yang's avatar
Michael Yang committed
1103
		cryptoPublicKey, cryptoPrivateKey, err := ed25519.GenerateKey(rand.Reader)
1104
1105
1106
1107
		if err != nil {
			return err
		}

Michael Yang's avatar
Michael Yang committed
1108
		privateKeyBytes, err := ssh.MarshalPrivateKey(cryptoPrivateKey, "")
1109
1110
1111
1112
		if err != nil {
			return err
		}

Michael Yang's avatar
Michael Yang committed
1113
		if err := os.MkdirAll(filepath.Dir(privKeyPath), 0o755); err != nil {
1114
1115
1116
			return fmt.Errorf("could not create directory %w", err)
		}

Michael Yang's avatar
Michael Yang committed
1117
		if err := os.WriteFile(privKeyPath, pem.EncodeToMemory(privateKeyBytes), 0o600); err != nil {
1118
1119
1120
			return err
		}

Michael Yang's avatar
Michael Yang committed
1121
		sshPublicKey, err := ssh.NewPublicKey(cryptoPublicKey)
1122
1123
1124
1125
		if err != nil {
			return err
		}

Michael Yang's avatar
Michael Yang committed
1126
		publicKeyBytes := ssh.MarshalAuthorizedKey(sshPublicKey)
1127

Michael Yang's avatar
Michael Yang committed
1128
		if err := os.WriteFile(pubKeyPath, publicKeyBytes, 0o644); err != nil {
1129
1130
1131
			return err
		}

Michael Yang's avatar
Michael Yang committed
1132
		fmt.Printf("Your new public key is: \n\n%s\n", publicKeyBytes)
1133
1134
1135
1136
	}
	return nil
}

Michael Yang's avatar
Michael Yang committed
1137
func checkServerHeartbeat(cmd *cobra.Command, _ []string) error {
Michael Yang's avatar
Michael Yang committed
1138
	client, err := api.ClientFromEnvironment()
1139
1140
1141
	if err != nil {
		return err
	}
Michael Yang's avatar
Michael Yang committed
1142
	if err := client.Heartbeat(cmd.Context()); err != nil {
1143
		if !strings.Contains(err.Error(), " refused") {
Bruce MacDonald's avatar
Bruce MacDonald committed
1144
1145
			return err
		}
1146
		if err := startApp(cmd.Context(), client); err != nil {
Michael Yang's avatar
lint  
Michael Yang committed
1147
			return errors.New("could not connect to ollama app, is it running?")
1148
1149
1150
1151
1152
		}
	}
	return nil
}

Michael Yang's avatar
Michael Yang committed
1153
1154
1155
1156
1157
1158
1159
1160
func versionHandler(cmd *cobra.Command, _ []string) {
	client, err := api.ClientFromEnvironment()
	if err != nil {
		return
	}

	serverVersion, err := client.Version(cmd.Context())
	if err != nil {
Michael Yang's avatar
Michael Yang committed
1161
1162
1163
1164
1165
		fmt.Println("Warning: could not connect to a running Ollama instance")
	}

	if serverVersion != "" {
		fmt.Printf("ollama version is %s\n", serverVersion)
Michael Yang's avatar
Michael Yang committed
1166
1167
	}

1168
	if serverVersion != version.Version {
Michael Yang's avatar
Michael Yang committed
1169
		fmt.Printf("Warning: client version is %s\n", version.Version)
1170
	}
Michael Yang's avatar
Michael Yang committed
1171
1172
}

1173
func appendEnvDocs(cmd *cobra.Command, envs []envconfig.EnvVar) {
1174
1175
1176
1177
1178
	if len(envs) == 0 {
		return
	}

	envUsage := `
1179
1180
Environment Variables:
`
1181
	for _, e := range envs {
1182
		envUsage += fmt.Sprintf("      %-24s   %s\n", e.Name, e.Description)
1183
1184
1185
	}

	cmd.SetUsageTemplate(cmd.UsageTemplate() + envUsage)
1186
1187
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1188
1189
func NewCLI() *cobra.Command {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
Michael Yang's avatar
Michael Yang committed
1190
	cobra.EnableCommandSorting = false
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1191

1192
	if runtime.GOOS == "windows" && term.IsTerminal(int(os.Stdout.Fd())) {
1193
		console.ConsoleFromFile(os.Stdin) //nolint:errcheck
1194
1195
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1196
	rootCmd := &cobra.Command{
1197
1198
1199
1200
		Use:           "ollama",
		Short:         "Large language model runner",
		SilenceUsage:  true,
		SilenceErrors: true,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1201
1202
1203
		CompletionOptions: cobra.CompletionOptions{
			DisableDefaultCmd: true,
		},
Michael Yang's avatar
Michael Yang committed
1204
1205
1206
1207
1208
1209
1210
1211
		Run: func(cmd *cobra.Command, args []string) {
			if version, _ := cmd.Flags().GetBool("version"); version {
				versionHandler(cmd, args)
				return
			}

			cmd.Print(cmd.UsageString())
		},
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1212
1213
	}

Michael Yang's avatar
Michael Yang committed
1214
	rootCmd.Flags().BoolP("version", "v", false, "Show version information")
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1215

1216
	createCmd := &cobra.Command{
1217
1218
		Use:     "create MODEL",
		Short:   "Create a model from a Modelfile",
Michael Yang's avatar
Michael Yang committed
1219
		Args:    cobra.ExactArgs(1),
1220
1221
		PreRunE: checkServerHeartbeat,
		RunE:    CreateHandler,
1222
1223
	}

1224
	createCmd.Flags().StringP("file", "f", "", "Name of the Modelfile (default \"Modelfile\"")
1225
	createCmd.Flags().StringP("quantize", "q", "", "Quantize model to this level (e.g. q4_0)")
1226

Patrick Devine's avatar
Patrick Devine committed
1227
1228
1229
	showCmd := &cobra.Command{
		Use:     "show MODEL",
		Short:   "Show information for a model",
Michael Yang's avatar
Michael Yang committed
1230
		Args:    cobra.ExactArgs(1),
Patrick Devine's avatar
Patrick Devine committed
1231
1232
1233
1234
1235
1236
1237
1238
		PreRunE: checkServerHeartbeat,
		RunE:    ShowHandler,
	}

	showCmd.Flags().Bool("license", false, "Show license of a model")
	showCmd.Flags().Bool("modelfile", false, "Show Modelfile of a model")
	showCmd.Flags().Bool("parameters", false, "Show parameters of a model")
	showCmd.Flags().Bool("template", false, "Show template of a model")
1239
	showCmd.Flags().Bool("system", false, "Show system message of a model")
1240
	showCmd.Flags().BoolP("verbose", "v", false, "Show detailed model information")
Patrick Devine's avatar
Patrick Devine committed
1241

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1242
	runCmd := &cobra.Command{
1243
1244
1245
1246
1247
		Use:     "run MODEL [PROMPT]",
		Short:   "Run a model",
		Args:    cobra.MinimumNArgs(1),
		PreRunE: checkServerHeartbeat,
		RunE:    RunHandler,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1248
1249
	}

1250
	runCmd.Flags().String("keepalive", "", "Duration to keep a model loaded (e.g. 5m)")
1251
	runCmd.Flags().Bool("verbose", false, "Show timings for response")
1252
	runCmd.Flags().Bool("insecure", false, "Use an insecure registry")
1253
	runCmd.Flags().Bool("nowordwrap", false, "Don't wrap words to the next line automatically")
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1254
	runCmd.Flags().String("format", "", "Response format (e.g. json)")
Patrick Devine's avatar
Patrick Devine committed
1255
1256
1257
1258
1259
1260
1261
1262
1263

	stopCmd := &cobra.Command{
		Use:     "stop MODEL",
		Short:   "Stop a running model",
		Args:    cobra.ExactArgs(1),
		PreRunE: checkServerHeartbeat,
		RunE:    StopHandler,
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1264
1265
1266
1267
	serveCmd := &cobra.Command{
		Use:     "serve",
		Aliases: []string{"start"},
		Short:   "Start ollama",
Michael Yang's avatar
Michael Yang committed
1268
		Args:    cobra.ExactArgs(0),
Michael Yang's avatar
Michael Yang committed
1269
		RunE:    RunServer,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1270
1271
	}

1272
	pullCmd := &cobra.Command{
1273
1274
		Use:     "pull MODEL",
		Short:   "Pull a model from a registry",
Michael Yang's avatar
Michael Yang committed
1275
		Args:    cobra.ExactArgs(1),
1276
1277
		PreRunE: checkServerHeartbeat,
		RunE:    PullHandler,
1278
1279
	}

1280
1281
	pullCmd.Flags().Bool("insecure", false, "Use an insecure registry")

1282
	pushCmd := &cobra.Command{
1283
1284
		Use:     "push MODEL",
		Short:   "Push a model to a registry",
Michael Yang's avatar
Michael Yang committed
1285
		Args:    cobra.ExactArgs(1),
1286
1287
		PreRunE: checkServerHeartbeat,
		RunE:    PushHandler,
1288
1289
	}

1290
1291
	pushCmd.Flags().Bool("insecure", false, "Use an insecure registry")

Patrick Devine's avatar
Patrick Devine committed
1292
	listCmd := &cobra.Command{
1293
		Use:     "list",
Patrick Devine's avatar
Patrick Devine committed
1294
		Aliases: []string{"ls"},
1295
		Short:   "List models",
1296
		PreRunE: checkServerHeartbeat,
1297
		RunE:    ListHandler,
1298
	}
1299
1300
1301
1302
1303
1304
1305
1306

	psCmd := &cobra.Command{
		Use:     "ps",
		Short:   "List running models",
		PreRunE: checkServerHeartbeat,
		RunE:    ListRunningHandler,
	}

Patrick Devine's avatar
Patrick Devine committed
1307
	copyCmd := &cobra.Command{
Michael Yang's avatar
Michael Yang committed
1308
		Use:     "cp SOURCE DESTINATION",
1309
		Short:   "Copy a model",
Michael Yang's avatar
Michael Yang committed
1310
		Args:    cobra.ExactArgs(2),
1311
1312
		PreRunE: checkServerHeartbeat,
		RunE:    CopyHandler,
Patrick Devine's avatar
Patrick Devine committed
1313
1314
	}

1315
	deleteCmd := &cobra.Command{
Michael Yang's avatar
Michael Yang committed
1316
		Use:     "rm MODEL [MODEL...]",
1317
1318
1319
1320
		Short:   "Remove a model",
		Args:    cobra.MinimumNArgs(1),
		PreRunE: checkServerHeartbeat,
		RunE:    DeleteHandler,
Patrick Devine's avatar
Patrick Devine committed
1321
1322
	}

1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
	runnerCmd := &cobra.Command{
		Use:    "runner",
		Hidden: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			return runner.Execute(os.Args[1:])
		},
		FParseErrWhitelist: cobra.FParseErrWhitelist{UnknownFlags: true},
	}
	runnerCmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
		_ = runner.Execute(args[1:])
	})

1335
1336
1337
	envVars := envconfig.AsMap()

	envs := []envconfig.EnvVar{envVars["OLLAMA_HOST"]}
1338

1339
1340
1341
1342
	for _, cmd := range []*cobra.Command{
		createCmd,
		showCmd,
		runCmd,
Patrick Devine's avatar
Patrick Devine committed
1343
		stopCmd,
1344
1345
1346
		pullCmd,
		pushCmd,
		listCmd,
1347
		psCmd,
1348
1349
		copyCmd,
		deleteCmd,
1350
		serveCmd,
1351
	} {
1352
1353
		switch cmd {
		case runCmd:
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
			appendEnvDocs(cmd, []envconfig.EnvVar{envVars["OLLAMA_HOST"], envVars["OLLAMA_NOHISTORY"]})
		case serveCmd:
			appendEnvDocs(cmd, []envconfig.EnvVar{
				envVars["OLLAMA_DEBUG"],
				envVars["OLLAMA_HOST"],
				envVars["OLLAMA_KEEP_ALIVE"],
				envVars["OLLAMA_MAX_LOADED_MODELS"],
				envVars["OLLAMA_MAX_QUEUE"],
				envVars["OLLAMA_MODELS"],
				envVars["OLLAMA_NUM_PARALLEL"],
				envVars["OLLAMA_NOPRUNE"],
				envVars["OLLAMA_ORIGINS"],
1366
				envVars["OLLAMA_SCHED_SPREAD"],
1367
				envVars["OLLAMA_TMPDIR"],
1368
				envVars["OLLAMA_FLASH_ATTENTION"],
1369
				envVars["OLLAMA_KV_CACHE_TYPE"],
1370
				envVars["OLLAMA_LLM_LIBRARY"],
1371
				envVars["OLLAMA_GPU_OVERHEAD"],
1372
				envVars["OLLAMA_LOAD_TIMEOUT"],
1373
			})
1374
1375
1376
		default:
			appendEnvDocs(cmd, envs)
		}
1377
1378
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1379
1380
	rootCmd.AddCommand(
		serveCmd,
1381
		createCmd,
Patrick Devine's avatar
Patrick Devine committed
1382
		showCmd,
1383
		runCmd,
Patrick Devine's avatar
Patrick Devine committed
1384
		stopCmd,
1385
1386
		pullCmd,
		pushCmd,
Patrick Devine's avatar
Patrick Devine committed
1387
		listCmd,
1388
		psCmd,
Patrick Devine's avatar
Patrick Devine committed
1389
		copyCmd,
1390
		deleteCmd,
1391
		runnerCmd,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1392
1393
1394
1395
	)

	return rootCmd
}