update runtime options (#864)

38616920 · Michael Yang · GitHub · 0d6342a8 · 38616920
Unverified Commit 38616920 authored Oct 20, 2023 by Michael Yang Committed by GitHub Oct 20, 2023
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 0 deletions

server/routes.go server/routes.go +4 -0

No files found.
--- a/server/routes.go
+++ b/server/routes.go
@@ -112,6 +112,10 @@ func load(ctx context.Context, workDir string, model *Model, reqOpts map[string]
 		loaded.Options = &opts
 	}

+	// update options for the loaded llm
+	// TODO(mxyng): this isn't thread safe, but it should be fine for now
+	loaded.runner.SetOptions(opts)
+
 	loaded.expireAt = time.Now().Add(sessionDuration)

 	if loaded.expireTimer == nil {