Skip to content

GitLab

Commit d4ebdadb authored by Jeffrey Morgan's avatar Jeffrey Morgan
Browse files

enable `cache_prompt` by default

parent c5f21f73
Hide whitespace changes
Inline Side-by-side
Showing with 2 additions and 3 deletions
api/types.go
View file @ d4ebdadb
...@@ -103,7 +103,6 @@ type Options struct { ...@@ -103,7 +103,6 @@ type Options struct {
MirostatEta float32 `json:"mirostat_eta,omitempty"` MirostatEta float32 `json:"mirostat_eta,omitempty"`
PenalizeNewline bool `json:"penalize_newline,omitempty"` PenalizeNewline bool `json:"penalize_newline,omitempty"`
Stop []string `json:"stop,omitempty"` Stop []string `json:"stop,omitempty"`
Cache bool `json:"cache,omitempty"`
} }
// Runner options which must be set when the model is loaded into memory // Runner options which must be set when the model is loaded into memory
......
llm/ext_server.go
View file @ d4ebdadb
...@@ -234,8 +234,8 @@ func predict(llm extServer, opts api.Options, ctx context.Context, predict Predi ...@@ -234,8 +234,8 @@ func predict(llm extServer, opts api.Options, ctx context.Context, predict Predi
"penalize_nl": opts.PenalizeNewline, "penalize_nl": opts.PenalizeNewline,
"seed": opts.Seed, "seed": opts.Seed,
"stop": opts.Stop, "stop": opts.Stop,
"image_data": imageData, "image_data": imageData,
"cache_prompt": opts.Cache, "cache_prompt": true,
} }
if predict.Format == "json" { if predict.Format == "json" {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment