cmd.go 25.7 KB
Newer Older
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1
2
3
package cmd

import (
Michael Yang's avatar
Michael Yang committed
4
	"bytes"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
5
	"context"
6
7
	"crypto/ed25519"
	"crypto/rand"
Michael Yang's avatar
Michael Yang committed
8
	"crypto/sha256"
9
	"encoding/pem"
Michael Yang's avatar
Michael Yang committed
10
	"errors"
Bruce MacDonald's avatar
Bruce MacDonald committed
11
	"fmt"
Michael Yang's avatar
Michael Yang committed
12
	"io"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
13
14
	"log"
	"net"
15
	"net/http"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
16
	"os"
17
	"os/exec"
18
	"os/signal"
19
	"path/filepath"
20
	"runtime"
Michael Yang's avatar
Michael Yang committed
21
	"strings"
22
	"syscall"
Michael Yang's avatar
Michael Yang committed
23
	"time"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
24

Patrick Devine's avatar
Patrick Devine committed
25
	"github.com/olekukonko/tablewriter"
Michael Yang's avatar
Michael Yang committed
26
	"github.com/spf13/cobra"
27
	"golang.org/x/crypto/ssh"
28
	"golang.org/x/term"
Michael Yang's avatar
Michael Yang committed
29

Jeffrey Morgan's avatar
Jeffrey Morgan committed
30
	"github.com/jmorganca/ollama/api"
Patrick Devine's avatar
Patrick Devine committed
31
	"github.com/jmorganca/ollama/format"
Michael Yang's avatar
Michael Yang committed
32
	"github.com/jmorganca/ollama/parser"
Michael Yang's avatar
Michael Yang committed
33
	"github.com/jmorganca/ollama/progress"
Patrick Devine's avatar
Patrick Devine committed
34
	"github.com/jmorganca/ollama/readline"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
35
	"github.com/jmorganca/ollama/server"
Michael Yang's avatar
Michael Yang committed
36
	"github.com/jmorganca/ollama/version"
Jeffrey Morgan's avatar
Jeffrey Morgan committed
37
38
)

39
func CreateHandler(cmd *cobra.Command, args []string) error {
40
	filename, _ := cmd.Flags().GetString("file")
41
42
43
44
45
	filename, err := filepath.Abs(filename)
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
46
	client, err := api.ClientFromEnvironment()
47
48
49
	if err != nil {
		return err
	}
50

Michael Yang's avatar
Michael Yang committed
51
52
53
54
55
	p := progress.NewProgress(os.Stderr)
	defer p.Stop()

	bars := make(map[string]*progress.Bar)

Michael Yang's avatar
Michael Yang committed
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
	modelfile, err := os.ReadFile(filename)
	if err != nil {
		return err
	}

	commands, err := parser.Parse(bytes.NewReader(modelfile))
	if err != nil {
		return err
	}

	home, err := os.UserHomeDir()
	if err != nil {
		return err
	}

71
72
	status := "transferring model data"
	spinner := progress.NewSpinner(status)
73
74
	p.Add(status, spinner)

Michael Yang's avatar
Michael Yang committed
75
76
77
78
79
80
81
82
83
84
	for _, c := range commands {
		switch c.Name {
		case "model", "adapter":
			path := c.Args
			if path == "~" {
				path = home
			} else if strings.HasPrefix(path, "~/") {
				path = filepath.Join(home, path[2:])
			}

85
86
87
88
			if !filepath.IsAbs(path) {
				path = filepath.Join(filepath.Dir(filename), path)
			}

Michael Yang's avatar
Michael Yang committed
89
90
			bin, err := os.Open(path)
			if errors.Is(err, os.ErrNotExist) && c.Name == "model" {
Michael Yang's avatar
Michael Yang committed
91
				continue
Michael Yang's avatar
Michael Yang committed
92
93
94
95
96
97
98
99
100
101
102
103
			} else if err != nil {
				return err
			}
			defer bin.Close()

			hash := sha256.New()
			if _, err := io.Copy(hash, bin); err != nil {
				return err
			}
			bin.Seek(0, io.SeekStart)

			digest := fmt.Sprintf("sha256:%x", hash.Sum(nil))
Michael Yang's avatar
Michael Yang committed
104
			if err = client.CreateBlob(cmd.Context(), digest, bin); err != nil {
Michael Yang's avatar
Michael Yang committed
105
106
107
				return err
			}

Michael Yang's avatar
Michael Yang committed
108
			modelfile = bytes.ReplaceAll(modelfile, []byte(c.Args), []byte("@"+digest))
Michael Yang's avatar
Michael Yang committed
109
110
		}
	}
Michael Yang's avatar
Michael Yang committed
111

112
	fn := func(resp api.ProgressResponse) error {
Michael Yang's avatar
Michael Yang committed
113
114
115
116
117
		if resp.Digest != "" {
			spinner.Stop()

			bar, ok := bars[resp.Digest]
			if !ok {
118
				bar = progress.NewBar(fmt.Sprintf("pulling %s...", resp.Digest[7:19]), resp.Total, resp.Completed)
Michael Yang's avatar
Michael Yang committed
119
120
121
122
123
124
125
126
127
128
129
130
131
				bars[resp.Digest] = bar
				p.Add(resp.Digest, bar)
			}

			bar.Set(resp.Completed)
		} else if status != resp.Status {
			spinner.Stop()

			status = resp.Status
			spinner = progress.NewSpinner(status)
			p.Add(status, spinner)
		}

132
133
134
		return nil
	}

135
	request := api.CreateRequest{Name: args[0], Modelfile: string(modelfile)}
136
137
138
139
140
141
142
	if err := client.Create(context.Background(), &request, fn); err != nil {
		return err
	}

	return nil
}

143
func RunHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
144
	client, err := api.ClientFromEnvironment()
145
146
147
148
	if err != nil {
		return err
	}

149
150
151
	name := args[0]
	// check if the model exists on the server
	_, err = client.Show(context.Background(), &api.ShowRequest{Name: name})
Michael Yang's avatar
Michael Yang committed
152
153
154
155
	var statusError api.StatusError
	switch {
	case errors.As(err, &statusError) && statusError.StatusCode == http.StatusNotFound:
		if err := PullHandler(cmd, args); err != nil {
156
			return err
Michael Yang's avatar
Michael Yang committed
157
		}
Michael Yang's avatar
Michael Yang committed
158
159
	case err != nil:
		return err
160
161
	}

Michael Yang's avatar
Michael Yang committed
162
	return RunGenerate(cmd, args)
Bruce MacDonald's avatar
Bruce MacDonald committed
163
164
}

165
func PushHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
166
	client, err := api.ClientFromEnvironment()
167
168
169
	if err != nil {
		return err
	}
170

171
172
173
174
175
	insecure, err := cmd.Flags().GetBool("insecure")
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
176
177
178
179
	p := progress.NewProgress(os.Stderr)
	defer p.Stop()

	bars := make(map[string]*progress.Bar)
180
181
	var status string
	var spinner *progress.Spinner
Michael Yang's avatar
Michael Yang committed
182

183
	fn := func(resp api.ProgressResponse) error {
Michael Yang's avatar
Michael Yang committed
184
		if resp.Digest != "" {
185
186
187
			if spinner != nil {
				spinner.Stop()
			}
Michael Yang's avatar
Michael Yang committed
188
189
190

			bar, ok := bars[resp.Digest]
			if !ok {
191
				bar = progress.NewBar(fmt.Sprintf("pushing %s...", resp.Digest[7:19]), resp.Total, resp.Completed)
Michael Yang's avatar
Michael Yang committed
192
193
194
195
196
197
				bars[resp.Digest] = bar
				p.Add(resp.Digest, bar)
			}

			bar.Set(resp.Completed)
		} else if status != resp.Status {
198
199
200
			if spinner != nil {
				spinner.Stop()
			}
Michael Yang's avatar
Michael Yang committed
201
202
203
204
205
206

			status = resp.Status
			spinner = progress.NewSpinner(status)
			p.Add(status, spinner)
		}

207
208
209
		return nil
	}

Michael Yang's avatar
Michael Yang committed
210
211
212
213
214
	request := api.PushRequest{Name: args[0], Insecure: insecure}
	if err := client.Push(context.Background(), &request, fn); err != nil {
		return err
	}

215
	spinner.Stop()
Michael Yang's avatar
Michael Yang committed
216
	return nil
217
218
}

219
func ListHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
220
	client, err := api.ClientFromEnvironment()
221
222
223
	if err != nil {
		return err
	}
Patrick Devine's avatar
Patrick Devine committed
224
225
226
227
228
229
230
231
232

	models, err := client.List(context.Background())
	if err != nil {
		return err
	}

	var data [][]string

	for _, m := range models.Models {
Michael Yang's avatar
Michael Yang committed
233
		if len(args) == 0 || strings.HasPrefix(m.Name, args[0]) {
234
			data = append(data, []string{m.Name, m.Digest[:12], format.HumanBytes(m.Size), format.HumanTime(m.ModifiedAt, "Never")})
Michael Yang's avatar
Michael Yang committed
235
		}
Patrick Devine's avatar
Patrick Devine committed
236
237
238
	}

	table := tablewriter.NewWriter(os.Stdout)
Patrick Devine's avatar
Patrick Devine committed
239
	table.SetHeader([]string{"NAME", "ID", "SIZE", "MODIFIED"})
Patrick Devine's avatar
Patrick Devine committed
240
241
242
243
244
245
246
247
248
249
250
251
	table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeaderLine(false)
	table.SetBorder(false)
	table.SetNoWhiteSpace(true)
	table.SetTablePadding("\t")
	table.AppendBulk(data)
	table.Render()

	return nil
}

252
func DeleteHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
253
	client, err := api.ClientFromEnvironment()
254
255
256
	if err != nil {
		return err
	}
257

258
259
260
261
262
263
	for _, name := range args {
		req := api.DeleteRequest{Name: name}
		if err := client.Delete(context.Background(), &req); err != nil {
			return err
		}
		fmt.Printf("deleted '%s'\n", name)
264
265
266
267
	}
	return nil
}

Patrick Devine's avatar
Patrick Devine committed
268
func ShowHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
269
	client, err := api.ClientFromEnvironment()
Patrick Devine's avatar
Patrick Devine committed
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
	if err != nil {
		return err
	}

	if len(args) != 1 {
		return errors.New("missing model name")
	}

	license, errLicense := cmd.Flags().GetBool("license")
	modelfile, errModelfile := cmd.Flags().GetBool("modelfile")
	parameters, errParams := cmd.Flags().GetBool("parameters")
	system, errSystem := cmd.Flags().GetBool("system")
	template, errTemplate := cmd.Flags().GetBool("template")

	for _, boolErr := range []error{errLicense, errModelfile, errParams, errSystem, errTemplate} {
		if boolErr != nil {
			return errors.New("error retrieving flags")
		}
	}

	flagsSet := 0
	showType := ""

	if license {
		flagsSet++
		showType = "license"
	}

	if modelfile {
		flagsSet++
		showType = "modelfile"
	}

	if parameters {
		flagsSet++
		showType = "parameters"
	}

	if system {
		flagsSet++
		showType = "system"
	}

	if template {
		flagsSet++
		showType = "template"
	}

	if flagsSet > 1 {
319
		return errors.New("only one of '--license', '--modelfile', '--parameters', '--system', or '--template' can be specified")
Patrick Devine's avatar
Patrick Devine committed
320
	} else if flagsSet == 0 {
321
		return errors.New("one of '--license', '--modelfile', '--parameters', '--system', or '--template' must be specified")
Patrick Devine's avatar
Patrick Devine committed
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
	}

	req := api.ShowRequest{Name: args[0]}
	resp, err := client.Show(context.Background(), &req)
	if err != nil {
		return err
	}

	switch showType {
	case "license":
		fmt.Println(resp.License)
	case "modelfile":
		fmt.Println(resp.Modelfile)
	case "parameters":
		fmt.Println(resp.Parameters)
	case "system":
		fmt.Println(resp.System)
	case "template":
		fmt.Println(resp.Template)
	}

	return nil
}

Patrick Devine's avatar
Patrick Devine committed
346
func CopyHandler(cmd *cobra.Command, args []string) error {
Michael Yang's avatar
Michael Yang committed
347
	client, err := api.ClientFromEnvironment()
348
349
350
	if err != nil {
		return err
	}
Patrick Devine's avatar
Patrick Devine committed
351
352
353
354
355
356
357
358
359

	req := api.CopyRequest{Source: args[0], Destination: args[1]}
	if err := client.Copy(context.Background(), &req); err != nil {
		return err
	}
	fmt.Printf("copied '%s' to '%s'\n", args[0], args[1])
	return nil
}

360
func PullHandler(cmd *cobra.Command, args []string) error {
361
362
363
364
365
	insecure, err := cmd.Flags().GetBool("insecure")
	if err != nil {
		return err
	}

Michael Yang's avatar
Michael Yang committed
366
	client, err := api.ClientFromEnvironment()
367
368
369
	if err != nil {
		return err
	}
370

Michael Yang's avatar
Michael Yang committed
371
372
373
374
375
	p := progress.NewProgress(os.Stderr)
	defer p.Stop()

	bars := make(map[string]*progress.Bar)

376
377
	var status string
	var spinner *progress.Spinner
Michael Yang's avatar
Michael Yang committed
378

379
	fn := func(resp api.ProgressResponse) error {
Michael Yang's avatar
Michael Yang committed
380
		if resp.Digest != "" {
381
382
383
			if spinner != nil {
				spinner.Stop()
			}
Michael Yang's avatar
Michael Yang committed
384
385
386

			bar, ok := bars[resp.Digest]
			if !ok {
387
				bar = progress.NewBar(fmt.Sprintf("pulling %s...", resp.Digest[7:19]), resp.Total, resp.Completed)
Michael Yang's avatar
Michael Yang committed
388
389
390
391
392
393
				bars[resp.Digest] = bar
				p.Add(resp.Digest, bar)
			}

			bar.Set(resp.Completed)
		} else if status != resp.Status {
394
395
396
			if spinner != nil {
				spinner.Stop()
			}
Michael Yang's avatar
Michael Yang committed
397
398
399
400
401
402

			status = resp.Status
			spinner = progress.NewSpinner(status)
			p.Add(status, spinner)
		}

403
404
		return nil
	}
405

Michael Yang's avatar
Michael Yang committed
406
407
408
409
410
411
	request := api.PullRequest{Name: args[0], Insecure: insecure}
	if err := client.Pull(context.Background(), &request, fn); err != nil {
		return err
	}

	return nil
Michael Yang's avatar
Michael Yang committed
412
413
}

414
func RunGenerate(cmd *cobra.Command, args []string) error {
415
416
417
418
419
420
421
422
	interactive := true

	opts := generateOptions{
		Model:    args[0],
		WordWrap: os.Getenv("TERM") == "xterm-256color",
		Options:  map[string]interface{}{},
	}

423
424
425
426
	format, err := cmd.Flags().GetString("format")
	if err != nil {
		return err
	}
427
	opts.Format = format
428

429
	prompts := args[1:]
430

431
432
433
	// prepend stdin to the prompt if provided
	if !term.IsTerminal(int(os.Stdin.Fd())) {
		in, err := io.ReadAll(os.Stdin)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
434
435
436
437
		if err != nil {
			return err
		}

438
		prompts = append([]string{string(in)}, prompts...)
439
440
		opts.WordWrap = false
		interactive = false
Michael Yang's avatar
Michael Yang committed
441
	}
442
443
444
	opts.Prompt = strings.Join(prompts, " ")
	if len(prompts) > 0 {
		interactive = false
Michael Yang's avatar
Michael Yang committed
445
446
	}

447
448
449
450
	nowrap, err := cmd.Flags().GetBool("nowordwrap")
	if err != nil {
		return err
	}
451
	opts.WordWrap = !nowrap
452

453
454
	if !interactive {
		return generate(cmd, opts)
455
456
	}

457
	return generateInteractive(cmd, opts)
Michael Yang's avatar
Michael Yang committed
458
459
}

Michael Yang's avatar
Michael Yang committed
460
type generateContextKey string
Michael Yang's avatar
Michael Yang committed
461

462
463
464
465
466
467
468
469
470
type generateOptions struct {
	Model    string
	Prompt   string
	WordWrap bool
	Format   string
	Options  map[string]interface{}
}

func generate(cmd *cobra.Command, opts generateOptions) error {
Michael Yang's avatar
Michael Yang committed
471
	client, err := api.ClientFromEnvironment()
Patrick Devine's avatar
Patrick Devine committed
472
473
474
	if err != nil {
		return err
	}
Michael Yang's avatar
Michael Yang committed
475

Michael Yang's avatar
Michael Yang committed
476
	p := progress.NewProgress(os.Stderr)
477
	defer p.StopAndClear()
Michael Yang's avatar
Michael Yang committed
478
479
480
481

	spinner := progress.NewSpinner("")
	p.Add("", spinner)

Patrick Devine's avatar
Patrick Devine committed
482
	var latest api.GenerateResponse
483

Patrick Devine's avatar
Patrick Devine committed
484
485
486
487
	generateContext, ok := cmd.Context().Value(generateContextKey("context")).([]int)
	if !ok {
		generateContext = []int{}
	}
Michael Yang's avatar
Michael Yang committed
488

489
	termWidth, _, err := term.GetSize(int(os.Stdout.Fd()))
490
	if err != nil {
491
		opts.WordWrap = false
492
493
	}

494
495
496
497
498
499
500
501
502
503
504
505
506
	cancelCtx, cancel := context.WithCancel(context.Background())
	defer cancel()

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT)
	var abort bool

	go func() {
		<-sigChan
		cancel()
		abort = true
	}()

507
508
509
	var currentLineLength int
	var wordBuffer string

510
511
512
513
514
515
516
	request := api.GenerateRequest{
		Model:   opts.Model,
		Prompt:  opts.Prompt,
		Context: generateContext,
		Format:  opts.Format,
		Options: opts.Options,
	}
Patrick Devine's avatar
Patrick Devine committed
517
	fn := func(response api.GenerateResponse) error {
Michael Yang's avatar
Michael Yang committed
518
519
		p.StopAndClear()

Patrick Devine's avatar
Patrick Devine committed
520
		latest = response
521

522
		if opts.WordWrap {
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
			for _, ch := range response.Response {
				if currentLineLength+1 > termWidth-5 {
					// backtrack the length of the last word and clear to the end of the line
					fmt.Printf("\x1b[%dD\x1b[K\n", len(wordBuffer))
					fmt.Printf("%s%c", wordBuffer, ch)
					currentLineLength = len(wordBuffer) + 1
				} else {
					fmt.Print(string(ch))
					currentLineLength += 1

					switch ch {
					case ' ':
						wordBuffer = ""
					case '\n':
						currentLineLength = 0
					default:
						wordBuffer += string(ch)
					}
				}
			}
		} else {
			fmt.Print(response.Response)
		}

Patrick Devine's avatar
Patrick Devine committed
547
548
		return nil
	}
549

550
	if err := client.Generate(cancelCtx, &request, fn); err != nil {
551
		if strings.Contains(err.Error(), "context canceled") && abort {
552
			return nil
553
		}
Patrick Devine's avatar
Patrick Devine committed
554
555
		return err
	}
556
	if opts.Prompt != "" {
Michael Yang's avatar
Michael Yang committed
557
558
		fmt.Println()
		fmt.Println()
Patrick Devine's avatar
Patrick Devine committed
559
	}
560

Patrick Devine's avatar
Patrick Devine committed
561
	if !latest.Done {
562
563
564
		if abort {
			return nil
		}
Patrick Devine's avatar
Patrick Devine committed
565
566
		return errors.New("unexpected end of response")
	}
567

Patrick Devine's avatar
Patrick Devine committed
568
569
570
571
	verbose, err := cmd.Flags().GetBool("verbose")
	if err != nil {
		return err
	}
Michael Yang's avatar
Michael Yang committed
572

Patrick Devine's avatar
Patrick Devine committed
573
574
	if verbose {
		latest.Summary()
Michael Yang's avatar
Michael Yang committed
575
	}
Michael Yang's avatar
Michael Yang committed
576

Patrick Devine's avatar
Patrick Devine committed
577
578
579
580
	ctx := cmd.Context()
	ctx = context.WithValue(ctx, generateContextKey("context"), latest.Context)
	cmd.SetContext(ctx)

Michael Yang's avatar
Michael Yang committed
581
582
583
	return nil
}

584
func generateInteractive(cmd *cobra.Command, opts generateOptions) error {
Patrick Devine's avatar
Patrick Devine committed
585
	// load the model
586
587
588
589
590
	loadOpts := generateOptions{
		Model:  opts.Model,
		Prompt: "",
	}
	if err := generate(cmd, loadOpts); err != nil {
Patrick Devine's avatar
Patrick Devine committed
591
592
593
		return err
	}

Michael Yang's avatar
Michael Yang committed
594
	usage := func() {
Patrick Devine's avatar
Patrick Devine committed
595
596
597
598
599
600
601
602
603
604
605
606
		fmt.Fprintln(os.Stderr, "Available Commands:")
		fmt.Fprintln(os.Stderr, "  /set         Set session variables")
		fmt.Fprintln(os.Stderr, "  /show        Show model information")
		fmt.Fprintln(os.Stderr, "  /bye         Exit")
		fmt.Fprintln(os.Stderr, "  /?, /help    Help for a command")
		fmt.Fprintln(os.Stderr, "")
		fmt.Fprintln(os.Stderr, "Use \"\"\" to begin a multi-line message.")
		fmt.Fprintln(os.Stderr, "")
	}

	usageSet := func() {
		fmt.Fprintln(os.Stderr, "Available Commands:")
607
		fmt.Fprintln(os.Stderr, "  /set parameter    Set a parameter")
Patrick Devine's avatar
Patrick Devine committed
608
609
610
611
		fmt.Fprintln(os.Stderr, "  /set history      Enable history")
		fmt.Fprintln(os.Stderr, "  /set nohistory    Disable history")
		fmt.Fprintln(os.Stderr, "  /set wordwrap     Enable wordwrap")
		fmt.Fprintln(os.Stderr, "  /set nowordwrap   Disable wordwrap")
Jeffrey Morgan's avatar
Jeffrey Morgan committed
612
613
		fmt.Fprintln(os.Stderr, "  /set format json  Enable JSON mode")
		fmt.Fprintln(os.Stderr, "  /set noformat     Disable formatting")
Patrick Devine's avatar
Patrick Devine committed
614
615
616
617
618
619
620
621
622
623
624
625
626
		fmt.Fprintln(os.Stderr, "  /set verbose      Show LLM stats")
		fmt.Fprintln(os.Stderr, "  /set quiet        Disable LLM stats")
		fmt.Fprintln(os.Stderr, "")
	}

	usageShow := func() {
		fmt.Fprintln(os.Stderr, "Available Commands:")
		fmt.Fprintln(os.Stderr, "  /show license      Show model license")
		fmt.Fprintln(os.Stderr, "  /show modelfile    Show Modelfile for this model")
		fmt.Fprintln(os.Stderr, "  /show parameters   Show parameters for this model")
		fmt.Fprintln(os.Stderr, "  /show system       Show system prompt")
		fmt.Fprintln(os.Stderr, "  /show template     Show prompt template")
		fmt.Fprintln(os.Stderr, "")
Michael Yang's avatar
Michael Yang committed
627
628
	}

629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
	// only list out the most common parameters
	usageParameters := func() {
		fmt.Fprintln(os.Stderr, "Available Parameters:")
		fmt.Fprintln(os.Stderr, "  /set parameter seed <int>             Random number seed")
		fmt.Fprintln(os.Stderr, "  /set parameter num_predict <int>      Max number of tokens to predict")
		fmt.Fprintln(os.Stderr, "  /set parameter top_k <int>            Pick from top k num of tokens")
		fmt.Fprintln(os.Stderr, "  /set parameter top_p <float>          Pick token based on sum of probabilities")
		fmt.Fprintln(os.Stderr, "  /set parameter num_ctx <int>          Set the context size")
		fmt.Fprintln(os.Stderr, "  /set parameter temperature <float>    Set creativity level")
		fmt.Fprintln(os.Stderr, "  /set parameter repeat_penalty <float> How strongly to penalize repetitions")
		fmt.Fprintln(os.Stderr, "  /set parameter repeat_last_n <int>    Set how far back to look for repetitions")
		fmt.Fprintln(os.Stderr, "  /set parameter num_gpu <int>          The number of layers to send to the GPU")
		fmt.Fprintln(os.Stderr, "  /set parameter stop \"<string>\", ...   Set the stop parameters")
		fmt.Fprintln(os.Stderr, "")
	}

645
	scanner, err := readline.New(readline.Prompt{
Patrick Devine's avatar
Patrick Devine committed
646
647
648
649
		Prompt:         ">>> ",
		AltPrompt:      "... ",
		Placeholder:    "Send a message (/? for help)",
		AltPlaceholder: `Use """ to end multi-line input`,
650
	})
Michael Yang's avatar
Michael Yang committed
651
652
653
654
	if err != nil {
		return err
	}

655
656
657
	fmt.Print(readline.StartBracketedPaste)
	defer fmt.Printf(readline.EndBracketedPaste)

658
	var prompt string
659

Michael Yang's avatar
Michael Yang committed
660
661
662
663
	for {
		line, err := scanner.Readline()
		switch {
		case errors.Is(err, io.EOF):
664
			fmt.Println()
Michael Yang's avatar
Michael Yang committed
665
666
			return nil
		case errors.Is(err, readline.ErrInterrupt):
667
			if line == "" {
Patrick Devine's avatar
Patrick Devine committed
668
				fmt.Println("\nUse Ctrl-D or /bye to exit.")
669
670
			}

671
672
673
			scanner.Prompt.UseAlt = false
			prompt = ""

Michael Yang's avatar
Michael Yang committed
674
675
			continue
		case err != nil:
Michael Yang's avatar
Michael Yang committed
676
677
678
			return err
		}

Michael Yang's avatar
Michael Yang committed
679
		switch {
680
681
682
683
684
685
686
		case strings.HasPrefix(prompt, `"""`):
			// if the prompt so far starts with """ then we're in multiline mode
			// and we need to keep reading until we find a line that ends with """
			cut, found := strings.CutSuffix(line, `"""`)
			prompt += cut + "\n"

			if !found {
687
688
				continue
			}
689
690
691
692

			prompt = strings.TrimPrefix(prompt, `"""`)
			scanner.Prompt.UseAlt = false
		case strings.HasPrefix(line, `"""`) && len(prompt) == 0:
Patrick Devine's avatar
Patrick Devine committed
693
			scanner.Prompt.UseAlt = true
694
695
696
697
			prompt += line + "\n"
			continue
		case scanner.Pasting:
			prompt += line + "\n"
698
			continue
Michael Yang's avatar
Michael Yang committed
699
700
		case strings.HasPrefix(line, "/list"):
			args := strings.Fields(line)
701
			if err := ListHandler(cmd, args[1:]); err != nil {
Michael Yang's avatar
Michael Yang committed
702
703
704
705
706
707
708
709
710
711
				return err
			}
		case strings.HasPrefix(line, "/set"):
			args := strings.Fields(line)
			if len(args) > 1 {
				switch args[1] {
				case "history":
					scanner.HistoryEnable()
				case "nohistory":
					scanner.HistoryDisable()
712
				case "wordwrap":
713
					opts.WordWrap = true
714
715
					fmt.Println("Set 'wordwrap' mode.")
				case "nowordwrap":
716
					opts.WordWrap = false
717
					fmt.Println("Set 'nowordwrap' mode.")
Michael Yang's avatar
Michael Yang committed
718
719
				case "verbose":
					cmd.Flags().Set("verbose", "true")
720
					fmt.Println("Set 'verbose' mode.")
Michael Yang's avatar
Michael Yang committed
721
722
				case "quiet":
					cmd.Flags().Set("verbose", "false")
723
					fmt.Println("Set 'quiet' mode.")
Jeffrey Morgan's avatar
Jeffrey Morgan committed
724
725
726
727
				case "format":
					if len(args) < 3 || args[2] != "json" {
						fmt.Println("Invalid or missing format. For 'json' mode use '/set format json'")
					} else {
728
						opts.Format = args[2]
Jeffrey Morgan's avatar
Jeffrey Morgan committed
729
730
731
						fmt.Printf("Set format to '%s' mode.\n", args[2])
					}
				case "noformat":
732
					opts.Format = ""
Jeffrey Morgan's avatar
Jeffrey Morgan committed
733
					fmt.Println("Disabled format.")
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
				case "parameter":
					if len(args) < 4 {
						usageParameters()
						continue
					}
					var params []string
					for _, p := range args[3:] {
						params = append(params, p)
					}
					fp, err := api.FormatParams(map[string][]string{args[2]: params})
					if err != nil {
						fmt.Printf("Couldn't set parameter: %q\n\n", err)
						continue
					}
					fmt.Printf("Set parameter '%s' to '%s'\n\n", args[2], strings.Join(params, ", "))
					opts.Options[args[2]] = fp[args[2]]
750
751
				default:
					fmt.Printf("Unknown command '/set %s'. Type /? for help\n", args[1])
752
753
				}
			} else {
Patrick Devine's avatar
Patrick Devine committed
754
				usageSet()
755
756
757
758
			}
		case strings.HasPrefix(line, "/show"):
			args := strings.Fields(line)
			if len(args) > 1 {
759
760
761
762
763
				client, err := api.ClientFromEnvironment()
				if err != nil {
					fmt.Println("error: couldn't connect to ollama server")
					return err
				}
764
				resp, err := client.Show(cmd.Context(), &api.ShowRequest{Name: opts.Model})
765
				if err != nil {
Patrick Devine's avatar
Patrick Devine committed
766
					fmt.Println("error: couldn't get model")
767
					return err
768
				}
Patrick Devine's avatar
Patrick Devine committed
769

770
771
				switch args[1] {
				case "license":
772
					if resp.License == "" {
773
						fmt.Print("No license was specified for this model.\n\n")
774
775
776
					} else {
						fmt.Println(resp.License)
					}
Patrick Devine's avatar
Patrick Devine committed
777
778
779
				case "modelfile":
					fmt.Println(resp.Modelfile)
				case "parameters":
780
					if resp.Parameters == "" {
781
						fmt.Print("No parameters were specified for this model.\n\n")
782
					} else {
783
784
785
786
787
788
789
790
						if len(opts.Options) > 0 {
							fmt.Println("User defined parameters:")
							for k, v := range opts.Options {
								fmt.Printf("%-*s %v\n", 30, k, v)
							}
							fmt.Println()
						}
						fmt.Println("Model defined parameters:")
791
792
						fmt.Println(resp.Parameters)
					}
793
				case "system":
794
					if resp.System == "" {
795
						fmt.Print("No system prompt was specified for this model.\n\n")
796
797
798
					} else {
						fmt.Println(resp.System)
					}
799
				case "template":
800
					if resp.Template == "" {
801
						fmt.Print("No prompt template was specified for this model.\n\n")
802
803
804
					} else {
						fmt.Println(resp.Template)
					}
805
				default:
806
					fmt.Printf("Unknown command '/show %s'. Type /? for help\n", args[1])
Michael Yang's avatar
Michael Yang committed
807
				}
Patrick Devine's avatar
Patrick Devine committed
808
809
810
811
812
813
814
815
816
817
818
819
			} else {
				usageShow()
			}
		case strings.HasPrefix(line, "/help"), strings.HasPrefix(line, "/?"):
			args := strings.Fields(line)
			if len(args) > 1 {
				switch args[1] {
				case "set", "/set":
					usageSet()
				case "show", "/show":
					usageShow()
				}
820
821
			} else {
				usage()
Michael Yang's avatar
Michael Yang committed
822
823
824
			}
		case line == "/exit", line == "/bye":
			return nil
825
826
827
		case strings.HasPrefix(line, "/"):
			args := strings.Fields(line)
			fmt.Printf("Unknown command '%s'. Type /? for help\n", args[0])
828
829
830
			continue
		default:
			prompt += line
Michael Yang's avatar
Michael Yang committed
831
832
		}

833
		if len(prompt) > 0 && prompt[0] != '/' {
834
835
			opts.Prompt = prompt
			if err := generate(cmd, opts); err != nil {
Patrick Devine's avatar
Patrick Devine committed
836
837
				return err
			}
838
839

			prompt = ""
Michael Yang's avatar
Michael Yang committed
840
841
		}
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
842
843
}

844
func RunServer(cmd *cobra.Command, _ []string) error {
Michael Yang's avatar
Michael Yang committed
845
846
847
	host, port, err := net.SplitHostPort(os.Getenv("OLLAMA_HOST"))
	if err != nil {
		host, port = "127.0.0.1", "11434"
Michael Yang's avatar
Michael Yang committed
848
		if ip := net.ParseIP(strings.Trim(os.Getenv("OLLAMA_HOST"), "[]")); ip != nil {
Michael Yang's avatar
Michael Yang committed
849
850
			host = ip.String()
		}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
851
	}
852

Michael Yang's avatar
Michael Yang committed
853
	if err := initializeKeypair(); err != nil {
854
855
856
		return err
	}

Michael Yang's avatar
Michael Yang committed
857
	ln, err := net.Listen("tcp", net.JoinHostPort(host, port))
858
859
860
	if err != nil {
		return err
	}
Jeffrey Morgan's avatar
Jeffrey Morgan committed
861

862
863
864
865
866
	var origins []string
	if o := os.Getenv("OLLAMA_ORIGINS"); o != "" {
		origins = strings.Split(o, ",")
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
867
	return server.Serve(ln, origins)
Jeffrey Morgan's avatar
Jeffrey Morgan committed
868
869
}

870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
func initializeKeypair() error {
	home, err := os.UserHomeDir()
	if err != nil {
		return err
	}

	privKeyPath := filepath.Join(home, ".ollama", "id_ed25519")
	pubKeyPath := filepath.Join(home, ".ollama", "id_ed25519.pub")

	_, err = os.Stat(privKeyPath)
	if os.IsNotExist(err) {
		fmt.Printf("Couldn't find '%s'. Generating new private key.\n", privKeyPath)
		_, privKey, err := ed25519.GenerateKey(rand.Reader)
		if err != nil {
			return err
		}

		privKeyBytes, err := format.OpenSSHPrivateKey(privKey, "")
		if err != nil {
			return err
		}

Michael Yang's avatar
Michael Yang committed
892
		err = os.MkdirAll(filepath.Dir(privKeyPath), 0o755)
893
894
895
896
		if err != nil {
			return fmt.Errorf("could not create directory %w", err)
		}

897
		err = os.WriteFile(privKeyPath, pem.EncodeToMemory(privKeyBytes), 0o600)
898
899
900
901
902
903
904
905
906
907
908
		if err != nil {
			return err
		}

		sshPrivateKey, err := ssh.NewSignerFromKey(privKey)
		if err != nil {
			return err
		}

		pubKeyData := ssh.MarshalAuthorizedKey(sshPrivateKey.PublicKey())

909
		err = os.WriteFile(pubKeyPath, pubKeyData, 0o644)
910
911
912
913
914
915
916
917
918
		if err != nil {
			return err
		}

		fmt.Printf("Your new public key is: \n\n%s\n", string(pubKeyData))
	}
	return nil
}

Bruce MacDonald's avatar
Bruce MacDonald committed
919
func startMacApp(client *api.Client) error {
Bruce MacDonald's avatar
Bruce MacDonald committed
920
921
922
923
924
	exe, err := os.Executable()
	if err != nil {
		return err
	}
	link, err := os.Readlink(exe)
Bruce MacDonald's avatar
Bruce MacDonald committed
925
926
927
	if err != nil {
		return err
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
928
929
930
	if !strings.Contains(link, "Ollama.app") {
		return fmt.Errorf("could not find ollama app")
	}
Bruce MacDonald's avatar
Bruce MacDonald committed
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
	path := strings.Split(link, "Ollama.app")
	if err := exec.Command("/usr/bin/open", "-a", path[0]+"Ollama.app").Run(); err != nil {
		return err
	}
	// wait for the server to start
	timeout := time.After(5 * time.Second)
	tick := time.Tick(500 * time.Millisecond)
	for {
		select {
		case <-timeout:
			return errors.New("timed out waiting for server to start")
		case <-tick:
			if err := client.Heartbeat(context.Background()); err == nil {
				return nil // server has started
			}
		}
	}
}

950
func checkServerHeartbeat(_ *cobra.Command, _ []string) error {
Michael Yang's avatar
Michael Yang committed
951
	client, err := api.ClientFromEnvironment()
952
953
954
	if err != nil {
		return err
	}
955
	if err := client.Heartbeat(context.Background()); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
956
957
958
959
960
		if !strings.Contains(err.Error(), "connection refused") {
			return err
		}
		if runtime.GOOS == "darwin" {
			if err := startMacApp(client); err != nil {
Bruce MacDonald's avatar
Bruce MacDonald committed
961
				return fmt.Errorf("could not connect to ollama app, is it running?")
962
			}
Bruce MacDonald's avatar
Bruce MacDonald committed
963
		} else {
964
965
966
967
968
969
			return fmt.Errorf("could not connect to ollama server, run 'ollama serve' to start it")
		}
	}
	return nil
}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
970
971
972
973
func NewCLI() *cobra.Command {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	rootCmd := &cobra.Command{
974
975
976
977
		Use:           "ollama",
		Short:         "Large language model runner",
		SilenceUsage:  true,
		SilenceErrors: true,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
978
979
980
		CompletionOptions: cobra.CompletionOptions{
			DisableDefaultCmd: true,
		},
Michael Yang's avatar
Michael Yang committed
981
		Version: version.Version,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
982
983
984
985
	}

	cobra.EnableCommandSorting = false

986
	createCmd := &cobra.Command{
987
988
		Use:     "create MODEL",
		Short:   "Create a model from a Modelfile",
Michael Yang's avatar
Michael Yang committed
989
		Args:    cobra.ExactArgs(1),
990
991
		PreRunE: checkServerHeartbeat,
		RunE:    CreateHandler,
992
993
994
995
	}

	createCmd.Flags().StringP("file", "f", "Modelfile", "Name of the Modelfile (default \"Modelfile\")")

Patrick Devine's avatar
Patrick Devine committed
996
997
998
	showCmd := &cobra.Command{
		Use:     "show MODEL",
		Short:   "Show information for a model",
Michael Yang's avatar
Michael Yang committed
999
		Args:    cobra.ExactArgs(1),
Patrick Devine's avatar
Patrick Devine committed
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
		PreRunE: checkServerHeartbeat,
		RunE:    ShowHandler,
	}

	showCmd.Flags().Bool("license", false, "Show license of a model")
	showCmd.Flags().Bool("modelfile", false, "Show Modelfile of a model")
	showCmd.Flags().Bool("parameters", false, "Show parameters of a model")
	showCmd.Flags().Bool("template", false, "Show template of a model")
	showCmd.Flags().Bool("system", false, "Show system prompt of a model")

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1010
	runCmd := &cobra.Command{
1011
1012
1013
1014
1015
		Use:     "run MODEL [PROMPT]",
		Short:   "Run a model",
		Args:    cobra.MinimumNArgs(1),
		PreRunE: checkServerHeartbeat,
		RunE:    RunHandler,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1016
1017
	}

1018
	runCmd.Flags().Bool("verbose", false, "Show timings for response")
1019
	runCmd.Flags().Bool("insecure", false, "Use an insecure registry")
1020
	runCmd.Flags().Bool("nowordwrap", false, "Don't wrap words to the next line automatically")
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1021
	runCmd.Flags().String("format", "", "Response format (e.g. json)")
1022

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1023
1024
1025
1026
	serveCmd := &cobra.Command{
		Use:     "serve",
		Aliases: []string{"start"},
		Short:   "Start ollama",
Michael Yang's avatar
Michael Yang committed
1027
		Args:    cobra.ExactArgs(0),
Michael Yang's avatar
Michael Yang committed
1028
		RunE:    RunServer,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1029
1030
	}

1031
	pullCmd := &cobra.Command{
1032
1033
		Use:     "pull MODEL",
		Short:   "Pull a model from a registry",
Michael Yang's avatar
Michael Yang committed
1034
		Args:    cobra.ExactArgs(1),
1035
1036
		PreRunE: checkServerHeartbeat,
		RunE:    PullHandler,
1037
1038
	}

1039
1040
	pullCmd.Flags().Bool("insecure", false, "Use an insecure registry")

1041
	pushCmd := &cobra.Command{
1042
1043
		Use:     "push MODEL",
		Short:   "Push a model to a registry",
Michael Yang's avatar
Michael Yang committed
1044
		Args:    cobra.ExactArgs(1),
1045
1046
		PreRunE: checkServerHeartbeat,
		RunE:    PushHandler,
1047
1048
	}

1049
1050
	pushCmd.Flags().Bool("insecure", false, "Use an insecure registry")

Patrick Devine's avatar
Patrick Devine committed
1051
	listCmd := &cobra.Command{
1052
		Use:     "list",
Patrick Devine's avatar
Patrick Devine committed
1053
		Aliases: []string{"ls"},
1054
		Short:   "List models",
1055
		PreRunE: checkServerHeartbeat,
1056
		RunE:    ListHandler,
1057
1058
	}

Patrick Devine's avatar
Patrick Devine committed
1059
	copyCmd := &cobra.Command{
Michael Yang's avatar
Michael Yang committed
1060
		Use:     "cp SOURCE TARGET",
1061
		Short:   "Copy a model",
Michael Yang's avatar
Michael Yang committed
1062
		Args:    cobra.ExactArgs(2),
1063
1064
		PreRunE: checkServerHeartbeat,
		RunE:    CopyHandler,
Patrick Devine's avatar
Patrick Devine committed
1065
1066
	}

1067
	deleteCmd := &cobra.Command{
Michael Yang's avatar
Michael Yang committed
1068
		Use:     "rm MODEL [MODEL...]",
1069
1070
1071
1072
		Short:   "Remove a model",
		Args:    cobra.MinimumNArgs(1),
		PreRunE: checkServerHeartbeat,
		RunE:    DeleteHandler,
Patrick Devine's avatar
Patrick Devine committed
1073
1074
	}

Jeffrey Morgan's avatar
Jeffrey Morgan committed
1075
1076
	rootCmd.AddCommand(
		serveCmd,
1077
		createCmd,
Patrick Devine's avatar
Patrick Devine committed
1078
		showCmd,
1079
		runCmd,
1080
1081
		pullCmd,
		pushCmd,
Patrick Devine's avatar
Patrick Devine committed
1082
		listCmd,
Patrick Devine's avatar
Patrick Devine committed
1083
		copyCmd,
1084
		deleteCmd,
Jeffrey Morgan's avatar
Jeffrey Morgan committed
1085
1086
1087
1088
	)

	return rootCmd
}