Skip to content

GitLab

"docker/Dockerfile.ci_gpu_cu11" did not exist on "92321347c06e43b9e6e35add8a5caec7dbf27b3f"
Commit b36b0b71 authored by Michael Yang's avatar Michael Yang
Browse files

use cut prefix

parent 094df375
Show whitespace changes
Inline Side-by-side
Showing with 4 additions and 6 deletions
llm/llama.go
View file @ b36b0b71
...@@ -531,16 +531,14 @@ func (llm *llama) Predict(ctx context.Context, prevContext []int, prompt string, ...@@ -531,16 +531,14 @@ func (llm *llama) Predict(ctx context.Context, prevContext []int, prompt string,
// This handles the request cancellation // This handles the request cancellation
return ctx.Err() return ctx.Err()
default: default:
line := scanner.Text() line := scanner.Bytes()
if line == "" { if len(line) == 0 {
continue continue
} }
// Read data from the server-side event stream if evt, ok := bytes.CutPrefix(line, []byte("data: ")); ok {
if strings.HasPrefix(line, "data: ") {
evt := line[6:]
var p prediction var p prediction
if err := json.Unmarshal([]byte(evt), &p); err != nil { if err := json.Unmarshal(evt, &p); err != nil {
return fmt.Errorf("error unmarshaling llm prediction response: %v", err) return fmt.Errorf("error unmarshaling llm prediction response: %v", err)
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment