Update sampling_params.md

876db8dc · Lianmin Zheng · GitHub · ad82bac6 · 876db8dc
Unverified Commit 876db8dc authored Jan 31, 2024 by Lianmin Zheng Committed by GitHub Jan 31, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 6 additions and 3 deletions

docs/sampling_params.md docs/sampling_params.md +6 -3

No files found.
--- a/docs/sampling_params.md
+++ b/docs/sampling_params.md
@@ -84,9 +84,12 @@ response = requests.post(
 )
 prev = 0
-for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"):
+for chunk in response.iter_lines(decode_unicode=False):
-    if chunk:
+    chunk = chunk.decode("utf-8")
-        data = json.loads(chunk.decode())
+    if chunk and chunk.startswith("data:"):
+        if chunk == "data: [DONE]":
+            break
+        data = json.loads(chunk[5:].strip("\n"))
        output = data["text"].strip()
        print(output[prev:], end="", flush=True)
        prev = len(output)