Apply 01-cache.diff

0a0e9f3e · Daniel Hiltgen · 58d95cc9 · 0a0e9f3e
Commit 0a0e9f3e authored Mar 19, 2024 by Daniel Hiltgen
Hide whitespace changes
Inline Side-by-side

Showing with 3 additions and 1 deletion

llm/ext_server/server.cpp llm/ext_server/server.cpp +3 -1

No files found.
--- a/llm/ext_server/server.cpp
+++ b/llm/ext_server/server.cpp
@@ -1007,13 +1007,15 @@ struct llama_server_context
                slot.n_sent_text += result.text_to_send.size();
                // add the token to slot queue and cache
            }
-            slot.add_token_string(result);
+
            if (slot.params.stream)
            {
                send_partial_response(slot, result);
            }
        }

+        slot.add_token_string(result);
+
        if (incomplete)
        {
            slot.has_next_token = true;