Unverified Commit 0c1041ad authored by Bruce MacDonald's avatar Bruce MacDonald Committed by GitHub
Browse files

runner: default to greedy sampler for performance (#9407)

As are adding support for weighted sampling we have seen some performance
regressions, bypassing the sampler logic for now and defaulting to greedy
until we can benchmark the new sampler logic.
parent c245b040
...@@ -575,23 +575,11 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) { ...@@ -575,23 +575,11 @@ func (s *Server) completion(w http.ResponseWriter, r *http.Request) {
return return
} }
sampler, err := sample.NewSampler(
req.Temperature,
req.TopK,
req.TopP,
req.MinP,
req.Seed,
)
if err != nil {
http.Error(w, fmt.Sprintf("Failed to create sampler: %v", err), http.StatusInternalServerError)
return
}
seq, err := s.NewSequence(req.Prompt, req.Images, NewSequenceParams{ seq, err := s.NewSequence(req.Prompt, req.Images, NewSequenceParams{
numPredict: req.NumPredict, numPredict: req.NumPredict,
stop: req.Stop, stop: req.Stop,
numKeep: int32(req.NumKeep), numKeep: int32(req.NumKeep),
sampler: sampler, sampler: sample.Greedy(), // TODO: add support for different samplers when performance is optimized
embedding: false, embedding: false,
}) })
if err != nil { if err != nil {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment