feat: OpenAI compatible http service (#123)

Signed-off-by: Ryan Olson <ryanolson@users.noreply.github.com>
Co-authored-by: Ryan McCormick <rmccormick@nvidia.com>
Co-authored-by: Neelay Shah <neelays@nvidia.com>

llm/rust/triton-llm/tests/data/replays/meta/llama-3.1-8b-instruct/completions/replay.yaml

+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- model_name: meta/llama-3.1-8b-instruct
+  file_path: completion.streaming.1
+  query: What is deep learning?
+  query_type: CompletionsStreaming
+
+- model_name: meta/llama-3.1-8b-instruct
+  file_path: completion.streaming.2
+  query: Write a limerick about the wonders of GPU computing.
+  query_type: CompletionsStreaming
+
+- model_name: meta/llama-3.1-8b-instruct
+  file_path: completion.streaming.3
+  query: Tell me about Dumbledore.
+  query_type: CompletionsStreaming

llm/rust/triton-llm/tests/data/replays/mistralai/mistral-7b-instruct-v0.3/completions/completion.streaming.1

+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"How","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" is","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" it","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" different","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" from","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" machine","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" learning","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"?","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"Deep","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" learning","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" is","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" a","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" subset","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" of","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" machine","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" learning","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" but","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" it","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" has","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" a","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" layer","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" of","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" complexity","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" that","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" involves","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" creating","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" models","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" with","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" multiple","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" layers","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" of","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" comput","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"ations","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" inspired","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" by","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" the","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" architecture","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" of","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" the","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" human","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" brain","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":".","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" In","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" a","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" traditional","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" machine","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" learning","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" model","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" we","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" present","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" the","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" data","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" and","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" which","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" answers","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" are","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" correct","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-771ade228a45417382478e6a165805ad","object":"text_completion","created":1729924105,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" and","logprobs":null,"finish_reason":"length","stop_reason":null}],"usage":null}
+
+data: [DONE]
+

llm/rust/triton-llm/tests/data/replays/mistralai/mistral-7b-instruct-v0.3/completions/completion.streaming.2

+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"In","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" a","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" world","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" where","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" numbers","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" take","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" flight","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"A","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" graph","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"ical","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" power","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" pure","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" white","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":";","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"N","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"VID","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"IA","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"'","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"s","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" dream","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"Op","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"ens","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" a","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" seam","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"For","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" science","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" and","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" art","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" light","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" as","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" a","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" k","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"ite","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":".","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"GPU","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" the","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" super","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"hero","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" un","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"ite","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"!","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"John","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" von","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" Ne","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"um","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"ann","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-7e8d736c1a8942d09a2268af9e29539a","object":"text_completion","created":1729924061,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" would","logprobs":null,"finish_reason":"length","stop_reason":null}],"usage":null}
+
+data: [DONE]
+

llm/rust/triton-llm/tests/data/replays/mistralai/mistral-7b-instruct-v0.3/completions/completion.streaming.3

+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"You","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" know","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" the","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" head","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"master","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" of","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" H","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"og","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"w","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"arts","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" in","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" your","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" books","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":".","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"\n","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"Oh","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" of","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" course","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":".","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" D","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"umbled","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"ore","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" is","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" one","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" of","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" the","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" most","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" beloved","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" characters","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" in","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" the","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" Harry","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" Pot","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"ter","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" series","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":".","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" He","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"'","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"s","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" a","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" powerful","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":",","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" yet","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" wise","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" and","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" kind","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" w","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"izard","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" who","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" serves","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" as","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" a","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" ment","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"or","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" and","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":"father","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" figure","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" to","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" Harry","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":".","logprobs":null,"finish_reason":null,"stop_reason":null}],"usage":null}
+
+data: {"id":"cmpl-a76fe4522b1a4cf6b4ebdc813b090f02","object":"text_completion","created":1729924141,"model":"mistralai/mistral-7b-instruct-v0.3","choices":[{"index":0,"text":" He","logprobs":null,"finish_reason":"length","stop_reason":null}],"usage":null}
+
+data: [DONE]
+

llm/rust/triton-llm/tests/data/replays/mistralai/mistral-7b-instruct-v0.3/completions/replay.yaml

+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- model_name: mistralai/mistral-7b-instruct-v0.3
+  file_path: completion.streaming.1
+  query: What is deep learning?
+  query_type: CompletionsStreaming
+
+- model_name: mistralai/mistral-7b-instruct-v0.3t
+  file_path: completion.streaming.2
+  query: Write a limerick about the wonders of GPU computing.
+  query_type: CompletionsStreaming
+
+- model_name: mistralai/mistral-7b-instruct-v0.3
+  file_path: completion.streaming.3
+  query: Tell me about Dumbledore.
+  query_type: CompletionsStreaming

llm/rust/triton-llm/tests/data/replays/mistralai/mixtral-8x22b-instruct-v0.1/chat_completions/chat-completion.streaming.1

+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":"assistant","content":null},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"Deep"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" learning"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" is"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" a"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" subset"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" of"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" machine"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" learning"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":","},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" which"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" is"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" essentially"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" a"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" neural"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" network"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" with"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" three"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" or"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" more"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" layers"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"."},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" These"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" neural"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" networks"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" attempt"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" to"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" sim"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ulate"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" the"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" behavior"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" of"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" the"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" human"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" brain"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"—"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"al"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"beit"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" far"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" from"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" matching"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" its"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" ability"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"—"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"in"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" order"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" to"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" \""},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"learn"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"\""},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" from"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" large"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" amounts"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" of"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" data"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"."},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" While"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" a"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" neural"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" network"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" with"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" a"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" single"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" layer"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-6430e81b825f4f779533bdfb3b9f37b3","object":"chat.completion.chunk","created":1728450669,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" can"},"logprobs":null,"finish_reason":"length","stop_reason":null}]}
+
+data: [DONE]
+

llm/rust/triton-llm/tests/data/replays/mistralai/mixtral-8x22b-instruct-v0.1/chat_completions/chat-completion.streaming.2

+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":"assistant","content":null},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"There"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" once"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" was"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" a"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" GPU"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" so"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" fine"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":","},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"\n"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"In"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" the"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" world"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" of"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" computing"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":","},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" it"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"'"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"d"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" shine"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"."},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"\n"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"With"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" its"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" parallel"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" threads"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":","},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"\n"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"It"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" swift"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ly"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" proceed"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"s"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":","},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"\n"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"Le"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"aving"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" old"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" CP"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"Us"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" far"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" behind"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"."},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-ead82b84735147aa97c96db53538b074","object":"chat.completion.chunk","created":1728450544,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":""},"logprobs":null,"finish_reason":"stop","stop_reason":null}]}
+
+data: [DONE]
+

llm/rust/triton-llm/tests/data/replays/mistralai/mixtral-8x22b-instruct-v0.1/chat_completions/chat-completion.streaming.3

+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":"assistant","content":null},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"Al"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"bus"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" Per"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"c"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ival"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" W"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ul"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"fr"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ic"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" Brian"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" D"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"umbled"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ore"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" is"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" a"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" fict"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ional"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" character"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" in"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" J"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"."},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"K"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"."},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" Row"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ling"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"'"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"s"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" Harry"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" Pot"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ter"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" series"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"."},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" He"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" is"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" the"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" Head"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"master"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" of"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" H"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"og"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"w"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"arts"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" School"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" of"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" W"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"itch"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"craft"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" and"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" W"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"izard"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ry"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" and"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" is"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" considered"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" one"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" of"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" the"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" most"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" powerful"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" w"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"iz"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":"ards"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" of"},"logprobs":null,"finish_reason":null}]}
+
+data: {"id":"chat-d026c528e68c43e582180425087c89ab","object":"chat.completion.chunk","created":1728450446,"model":"mistralai/mixtral-8x22b-instruct-v0.1","choices":[{"index":0,"delta":{"role":null,"content":" all"},"logprobs":null,"finish_reason":"length","stop_reason":null}]}
+
+data: [DONE]
+

llm/rust/triton-llm/tests/data/replays/mistralai/mixtral-8x22b-instruct-v0.1/chat_completions/replay.yaml

+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- model_name: mistralai/mixtral-8x22b-instruct-v0.1
+  file_path: chat-completion.streaming.1
+  query: What is deep learning?
+  query_type: ChatCompletionsStreaming
+
+- model_name: mistralai/mixtral-8x22b-instruct-v0.1
+  file_path: chat-completion.streaming.2
+  query: Write a limerick about the wonders of GPU computing.
+  query_type: ChatCompletionsStreaming
+
+- model_name: mistralai/mixtral-8x22b-instruct-v0.1
+  file_path: chat-completion.streaming.3
+  query: Tell me about Dumbledore.
+  query_type: ChatCompletionsStreaming

llm/rust/triton-llm/tests/data/replays/model_replay_config.yaml

+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+- model_name: meta-llama/llama-3.1-8b-instruct
+  file_path: meta/llama-3.1-8b-instruct/chat_completions/chat-completion.streaming.1
+  query: What is deep learning?
+  query_type: ChatCompletionsStreaming
+
+- model_name: meta-llama/llama-3.1-8b-instruct
+  file_path: meta/llama-3.1-8b-instruct/chat_completions/chat-completion.streaming.2
+  query: Write a limerick about the wonders of GPU computing.
+  query_type: ChatCompletionsStreaming
+
+- model_name: meta-llama/llama-3.1-8b-instruct
+  file_path: meta/llama-3.1-8b-instruct/chat_completions/chat-completion.streaming.3
+  query: Tell me about Dumbledore.
+  query_type: ChatCompletionsStreaming
+
+- model_name: mistralai/mixtral-8x22b-instruct-v0.1
+  file_path: mistralai/mixtral-8x22b-instruct-v0.1/chat_completions/chat-completion.streaming.1
+  query: What is deep learning?
+  query_type: ChatCompletionsStreaming
+
+- model_name: mistralai/mixtral-8x22b-instruct-v0.1
+  file_path: mistralai/mixtral-8x22b-instruct-v0.1/chat_completions/chat-completion.streaming.2
+  query: Write a limerick about the wonders of GPU computing.
+  query_type: ChatCompletionsStreaming
+
+- model_name: mistralai/mixtral-8x22b-instruct-v0.1
+  file_path: mistralai/mixtral-8x22b-instruct-v0.1/chat_completions/chat-completion.streaming.3
+  query: Tell me about Dumbledore.
+  query_type: ChatCompletionsStreaming
+
+- model_name: meta-llama/llama-3.1-8b-instruct
+  file_path: meta/llama-3.1-8b-instruct/completions/completion.streaming.1
+  query: Explain what is deep learning?
+  query_type: CompletionsStreaming
+
+- model_name: meta-llama/llama-3.1-8b-instruct
+  file_path: meta/llama-3.1-8b-instruct/completions/completion.streaming.2
+  query: Please write a limerick about the wonders of GPU computing.
+  query_type: CompletionsStreaming
+
+- model_name: meta-llama/llama-3.1-8b-instruct
+  file_path: meta/llama-3.1-8b-instruct/completions/completion.streaming.3
+  query: Tell me something about Dumbledore.
+  query_type: CompletionsStreaming
+
+- model_name: mistralai/mistral-7b-instruct-v0.3
+  file_path: mistralai/mistral-7b-instruct-v0.3/completions/completion.streaming.1
+  query: What is deep learning?
+  query_type: CompletionsStreaming
+
+- model_name: mistralai/mistral-7b-instruct-v0.3t
+  file_path: mistralai/mistral-7b-instruct-v0.3/completions/completion.streaming.2
+  query: Write a limerick about the wonders of GPU computing.
+  query_type: CompletionsStreaming
+
+- model_name: mistralai/mistral-7b-instruct-v0.3
+  file_path: mistralai/mistral-7b-instruct-v0.3/completions/completion.streaming.3
+  query: Tell me about Dumbledore.
+  query_type: CompletionsStreaming

llm/rust/triton-llm/tests/http-service.rs

+// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::Error;
+use async_stream::stream;
+use prometheus::{proto::MetricType, Registry};
+use reqwest::StatusCode;
+use std::sync::Arc;
+use triton_distributed::{
+    pipeline::{
+        async_trait, AsyncEngine, AsyncEngineContextProvider, ManyOut, ResponseStream, SingleIn,
+    },
+    CancellationToken,
+};
+use triton_llm::http::service::{
+    error::HttpError,
+    metrics::{Endpoint, RequestType, Status},
+    service_v2::HttpService,
+    Metrics,
+};
+use triton_llm::protocols::{
+    openai::{
+        chat_completions::{ChatCompletionRequest, ChatCompletionResponseDelta},
+        completions::{CompletionRequest, CompletionResponse},
+    },
+    Annotated,
+};
+
+struct CounterEngine {}
+
+#[async_trait]
+impl
+    AsyncEngine<
+        SingleIn<ChatCompletionRequest>,
+        ManyOut<Annotated<ChatCompletionResponseDelta>>,
+        Error,
+    > for CounterEngine
+{
+    async fn generate(
+        &self,
+        request: SingleIn<ChatCompletionRequest>,
+    ) -> Result<ManyOut<Annotated<ChatCompletionResponseDelta>>, Error> {
+        let (request, context) = request.transfer(());
+        let ctx = context.context();
+
+        let max_tokens = request.max_tokens.unwrap_or(0) as u64;
+
+        // let generator = ChatCompletionResponseDelta::generator(request.model.clone());
+        let generator = request.response_generator();
+
+        let stream = stream! {
+            tokio::time::sleep(std::time::Duration::from_millis(max_tokens)).await;
+            for i in 0..10 {
+                let choice = generator.create_choice(i as u64,Some(format!("choice {i}")), None, None);
+                yield Annotated::from_data(choice);
+            }
+        };
+
+        Ok(ResponseStream::new(Box::pin(stream), ctx))
+    }
+}
+
+struct AlwaysFailEngine {}
+
+#[async_trait]
+impl
+    AsyncEngine<
+        SingleIn<ChatCompletionRequest>,
+        ManyOut<Annotated<ChatCompletionResponseDelta>>,
+        Error,
+    > for AlwaysFailEngine
+{
+    async fn generate(
+        &self,
+        _request: SingleIn<ChatCompletionRequest>,
+    ) -> Result<ManyOut<Annotated<ChatCompletionResponseDelta>>, Error> {
+        Err(HttpError {
+            code: 403,
+            message: "Always fail".to_string(),
+        })?
+    }
+}
+
+#[async_trait]
+impl AsyncEngine<SingleIn<CompletionRequest>, ManyOut<Annotated<CompletionResponse>>, Error>
+    for AlwaysFailEngine
+{
+    async fn generate(
+        &self,
+        _request: SingleIn<CompletionRequest>,
+    ) -> Result<ManyOut<Annotated<CompletionResponse>>, Error> {
+        Err(HttpError {
+            code: 401,
+            message: "Always fail".to_string(),
+        })?
+    }
+}
+
+fn compare_counter(
+    metrics: Arc<Metrics>,
+    model: &str,
+    endpoint: &Endpoint,
+    request_type: &RequestType,
+    status: &Status,
+    expected: u64,
+) {
+    assert_eq!(
+        metrics.get_request_counter(model, endpoint, request_type, status),
+        expected,
+        "model: {}, endpoint: {:?}, request_type: {:?}, status: {:?}",
+        model,
+        endpoint.as_str(),
+        request_type.as_str(),
+        status.as_str()
+    );
+}
+
+fn compute_index(endpoint: &Endpoint, request_type: &RequestType, status: &Status) -> usize {
+    let endpoint = match endpoint {
+        Endpoint::Completions => 0,
+        Endpoint::ChatCompletions => 1,
+    };
+
+    let request_type = match request_type {
+        RequestType::Unary => 0,
+        RequestType::Stream => 1,
+    };
+
+    let status = match status {
+        Status::Success => 0,
+        Status::Error => 1,
+    };
+
+    endpoint * 4 + request_type * 2 + status
+}
+
+fn compare_counters(metrics: Arc<Metrics>, model: &str, expected: &[u64; 8]) {
+    for endpoint in &[Endpoint::Completions, Endpoint::ChatCompletions] {
+        for request_type in &[RequestType::Unary, RequestType::Stream] {
+            for status in &[Status::Success, Status::Error] {
+                let index = compute_index(endpoint, request_type, status);
+                compare_counter(
+                    metrics.clone(),
+                    model,
+                    endpoint,
+                    request_type,
+                    status,
+                    expected[index],
+                );
+            }
+        }
+    }
+}
+
+fn inc_counter(
+    endpoint: Endpoint,
+    request_type: RequestType,
+    status: Status,
+    expected: &mut [u64; 8],
+) {
+    let index = compute_index(&endpoint, &request_type, &status);
+    expected[index] += 1;
+}
+
+#[tokio::test]
+async fn test_http_service() {
+    let mut builder = HttpService::builder();
+    builder.port(8989);
+
+    let service = builder.build().unwrap();
+    let manager = service.model_manager().clone();
+
+    let token = CancellationToken::new();
+    let cancel_token = token.clone();
+    let task = tokio::spawn(async move { service.run(token.clone()).await });
+
+    let registry = Registry::new();
+
+    let counter = Arc::new(CounterEngine {});
+    let result = manager.add_chat_completions_model("foo", counter);
+    assert!(result.is_ok());
+
+    let failure = Arc::new(AlwaysFailEngine {});
+    let result = manager.add_chat_completions_model("bar", failure.clone());
+    assert!(result.is_ok());
+
+    let result = manager.add_completions_model("bar", failure);
+    assert!(result.is_ok());
+
+    let metrics = manager.metrics();
+    metrics.register(&registry).unwrap();
+
+    let mut foo_counters = [0u64; 8];
+    let mut bar_counters = [0u64; 8];
+
+    compare_counters(metrics.clone(), "foo", &foo_counters);
+    compare_counters(metrics.clone(), "bar", &bar_counters);
+
+    let client = reqwest::Client::new();
+
+    let mut request = ChatCompletionRequest::builder()
+        .model("foo")
+        .add_user_message("hi")
+        .build()
+        .unwrap();
+
+    // ==== ChatCompletions / Stream / Success ====
+    request.stream = Some(true);
+    request.max_tokens = Some(3000);
+
+    let response = client
+        .post("http://localhost:8989/v1/chat/completions")
+        .json(&request)
+        .send()
+        .await
+        .unwrap();
+
+    assert!(response.status().is_success(), "{:?}", response);
+
+    tokio::time::sleep(tokio::time::Duration::from_millis(1000)).await;
+    assert_eq!(metrics.get_inflight_count("foo"), 1);
+
+    // process byte stream
+    let _ = response.bytes().await.unwrap();
+
+    inc_counter(
+        Endpoint::ChatCompletions,
+        RequestType::Stream,
+        Status::Success,
+        &mut foo_counters,
+    );
+    compare_counters(metrics.clone(), "foo", &foo_counters);
+    compare_counters(metrics.clone(), "bar", &bar_counters);
+
+    // check registry and look or the request duration histogram
+    let families = registry.gather();
+    let histogram_metric_family = families
+        .into_iter()
+        .find(|m| m.get_name() == "nv_llm_http_service_request_duration_seconds")
+        .expect("Histogram metric not found");
+
+    assert_eq!(
+        histogram_metric_family.get_field_type(),
+        MetricType::HISTOGRAM
+    );
+
+    let histogram_metric = histogram_metric_family.get_metric();
+
+    assert_eq!(histogram_metric.len(), 1); // We have one metric with label model
+
+    let metric = &histogram_metric[0];
+    let histogram = metric.get_histogram();
+
+    let buckets = histogram.get_bucket();
+
+    let mut found = false;
+
+    for bucket in buckets {
+        let upper_bound = bucket.get_upper_bound();
+        let cumulative_count = bucket.get_cumulative_count();
+
+        println!(
+            "Bucket upper bound: {}, count: {}",
+            upper_bound, cumulative_count
+        );
+
+        // Since our observation is 2.5, it should fall into the bucket with upper bound 4.0
+        if upper_bound >= 4.0 {
+            assert_eq!(
+                cumulative_count, 1,
+                "Observation should be counted in the 4.0 bucket"
+            );
+            found = true;
+        } else {
+            assert_eq!(
+                cumulative_count, 0,
+                "No observations should be in this bucket"
+            );
+        }
+    }
+
+    assert!(found, "The expected bucket was not found");
+    // ==== ChatCompletions / Stream / Success ====
+
+    // ==== ChatCompletions / Unary / Success ====
+    request.stream = Some(false);
+    request.max_tokens = Some(0);
+
+    let future = client
+        .post("http://localhost:8989/v1/chat/completions")
+        .json(&request)
+        .send();
+
+    let response = future.await.unwrap();
+
+    assert!(response.status().is_success(), "{:?}", response);
+    inc_counter(
+        Endpoint::ChatCompletions,
+        RequestType::Unary,
+        Status::Success,
+        &mut foo_counters,
+    );
+    compare_counters(metrics.clone(), "foo", &foo_counters);
+    compare_counters(metrics.clone(), "bar", &bar_counters);
+    // ==== ChatCompletions / Unary / Success ====
+
+    // ==== ChatCompletions / Stream / Error ====
+    request.model = "bar".to_string();
+    request.max_tokens = Some(0);
+    request.stream = Some(true);
+
+    let response = client
+        .post("http://localhost:8989/v1/chat/completions")
+        .json(&request)
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::FORBIDDEN);
+    inc_counter(
+        Endpoint::ChatCompletions,
+        RequestType::Stream,
+        Status::Error,
+        &mut bar_counters,
+    );
+    compare_counters(metrics.clone(), "foo", &foo_counters);
+    compare_counters(metrics.clone(), "bar", &bar_counters);
+    // ==== ChatCompletions / Stream / Error ====
+
+    // ==== ChatCompletions / Unary / Error ====
+    request.stream = Some(false);
+
+    let response = client
+        .post("http://localhost:8989/v1/chat/completions")
+        .json(&request)
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::FORBIDDEN);
+    inc_counter(
+        Endpoint::ChatCompletions,
+        RequestType::Unary,
+        Status::Error,
+        &mut bar_counters,
+    );
+    compare_counters(metrics.clone(), "foo", &foo_counters);
+    compare_counters(metrics.clone(), "bar", &bar_counters);
+    // ==== ChatCompletions / Unary / Error ====
+
+    // ==== Completions / Unary / Error ====
+    let mut request = CompletionRequest::builder()
+        .model("bar")
+        .prompt("hi")
+        .build()
+        .unwrap();
+
+    let response = client
+        .post("http://localhost:8989/v1/completions")
+        .json(&request)
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+    inc_counter(
+        Endpoint::Completions,
+        RequestType::Unary,
+        Status::Error,
+        &mut bar_counters,
+    );
+    compare_counters(metrics.clone(), "foo", &foo_counters);
+    compare_counters(metrics.clone(), "bar", &bar_counters);
+    // ==== Completions / Unary / Error ====
+
+    // ==== Completions / Stream / Error ====
+    request.stream = Some(true);
+
+    let response = client
+        .post("http://localhost:8989/v1/completions")
+        .json(&request)
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
+    inc_counter(
+        Endpoint::Completions,
+        RequestType::Stream,
+        Status::Error,
+        &mut bar_counters,
+    );
+    compare_counters(metrics.clone(), "foo", &foo_counters);
+    compare_counters(metrics.clone(), "bar", &bar_counters);
+    // ==== Completions / Stream / Error ====
+
+    // =========== Test Invalid Request ===========
+    // send a completion request to a chat endpoint
+    request.stream = Some(false);
+
+    let response = client
+        .post("http://localhost:8989/v1/chat/completions")
+        .json(&request)
+        .send()
+        .await
+        .unwrap();
+
+    assert_eq!(
+        response.status(),
+        StatusCode::UNPROCESSABLE_ENTITY,
+        "{:?}",
+        response
+    );
+
+    // =========== Query /metrics endpoint ===========
+    let response = client
+        .get("http://localhost:8989/metrics")
+        .send()
+        .await
+        .unwrap();
+
+    assert!(response.status().is_success(), "{:?}", response);
+    println!("{}", response.text().await.unwrap());
+
+    cancel_token.cancel();
+    task.await.unwrap().unwrap();
+}

llm/rust/triton-llm/tests/openai_completions.rs

+// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use serde::{Deserialize, Serialize};
+use triton_llm::protocols::{
+    common,
+    openai::{
+        self,
+        completions::{CompletionRequest, CompletionRequestBuilder},
+        nvext::NvExt,
+    },
+};
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+struct CompletionSample {
+    request: CompletionRequest,
+    description: String,
+}
+
+impl CompletionSample {
+    fn new<F>(description: impl Into<String>, configure: F) -> Result<Self, String>
+    where
+        F: FnOnce(&mut CompletionRequestBuilder) -> &mut CompletionRequestBuilder,
+    {
+        let mut builder = CompletionRequestBuilder::default();
+        builder
+            .model("gpt-3.5-turbo")
+            .prompt("What is the meaning of life?");
+        configure(&mut builder);
+        Ok(Self {
+            request: builder.build().unwrap(),
+            description: description.into(),
+        })
+    }
+}
+
+#[test]
+fn minimum_viable_request() {
+    let request = CompletionRequest::builder()
+        .prompt("What is the meaning of life?")
+        .model("gpt-3.5-turbo")
+        .build()
+        .expect("error building request");
+
+    insta::assert_json_snapshot!(request);
+}
+
+#[test]
+fn missing_model() {
+    let request = CompletionRequest::builder()
+        .prompt("What is the meaning of life?")
+        .build();
+    assert!(request.is_err());
+}
+
+#[test]
+fn missing_prompt() {
+    let request = CompletionRequest::builder().model("gpt-3.5-turbo").build();
+    assert!(request.is_err());
+}
+
+#[test]
+fn out_of_range() {
+    let request = CompletionRequest::builder()
+        .prompt("What is the meaning of life?")
+        .model("gpt-3.5-turbo")
+        .temperature(openai::MAX_TEMPERATURE + 1.0)
+        .build();
+    assert!(request.is_err());
+
+    let request = CompletionRequest::builder()
+        .prompt("What is the meaning of life?")
+        .model("gpt-3.5-turbo")
+        .temperature(openai::MIN_TEMPERATURE - 1.0)
+        .build();
+    assert!(request.is_err());
+}
+
+#[test]
+fn ignore_eos() {
+    let request = CompletionRequest::builder()
+        .prompt("What is the meaning of life?")
+        .model("gpt-3.5-turbo")
+        .nvext(
+            NvExt::builder()
+                .ignore_eos(true)
+                .build()
+                .expect("error building nvext"),
+        )
+        .build()
+        .expect("error building request");
+
+    let request = common::CompletionRequest::try_from(request).expect("error converting request");
+
+    let ignore_eos = request.stop_conditions.ignore_eos.unwrap();
+    assert!(ignore_eos);
+}
+
+#[test]
+fn valid_samples() {
+    let mut settings = insta::Settings::clone_current();
+    settings.set_sort_maps(true);
+    let _guard = settings.bind_to_scope();
+
+    let samples = build_samples().expect("error building samples");
+
+    // iteration on all sample and call validate and expect it to be ok
+    for sample in &samples {
+        insta::with_settings!({
+            description => &sample.description,
+        }, {
+        insta::assert_json_snapshot!(sample.request);
+        });
+    }
+}
+#[allow(clippy::vec_init_then_push)]
+fn build_samples() -> Result<Vec<CompletionSample>, String> {
+    let mut samples = Vec::new();
+
+    samples.push(CompletionSample::new(
+        "should have only prompt and model fields",
+        |builder| builder,
+    )?);
+
+    samples.push(CompletionSample::new(
+        "should have prompt, model, and max_tokens fields",
+        |builder| builder.max_tokens(10),
+    )?);
+
+    samples.push(CompletionSample::new(
+        "should have prompt, model, and temperature fields",
+        |builder| builder.temperature(openai::MIN_TEMPERATURE),
+    )?);
+
+    samples.push(CompletionSample::new(
+        "should have prompt, model, and top_p fields",
+        |builder| builder.top_p(openai::MIN_TOP_P),
+    )?);
+
+    samples.push(CompletionSample::new(
+        "should have prompt, model, and frequency_penalty fields",
+        |builder| builder.frequency_penalty(openai::MIN_FREQUENCY_PENALTY),
+    )?);
+
+    samples.push(CompletionSample::new(
+        "should have prompt, model, and presence_penalty fields",
+        |builder| builder.presence_penalty(openai::MIN_PRESENCE_PENALTY),
+    )?);
+
+    samples.push(CompletionSample::new(
+        "should have prompt, model, and stop fields",
+        |builder| builder.stop(vec!["\n".to_string()]),
+    )?);
+
+    samples.push(CompletionSample::new(
+        "should have prompt, model, and echo fields",
+        |builder| builder.echo(true),
+    )?);
+
+    samples.push(CompletionSample::new(
+        "should have prompt, model, and stream fields",
+        |builder| builder.stream(true),
+    )?);
+
+    samples.push(CompletionSample::new(
+        "should have prompt, model, and logit_bias fields with the logits_bias having two key/value pairs",
+        |builder| builder.add_logit_bias(1337, -100).add_logit_bias("42", 100),
+    )?);
+
+    Ok(samples)
+}

llm/rust/triton-llm/tests/snapshots/openai_completions__minimum_viable_request.snap

+---
+source: triton-llm/tests/openai_completions.rs
+expression: request
+---
+{
+  "model": "gpt-3.5-turbo",
+  "prompt": "What is the meaning of life?"
+}

llm/rust/triton-llm/tests/snapshots/openai_completions__valid_samples-10.snap

+---
+source: triton-llm/tests/openai_completions.rs
+description: "should have prompt, model, and logit_bias fields with the logits_bias having two key/value pairs"
+expression: sample.request
+---
+{
+  "model": "gpt-3.5-turbo",
+  "prompt": "What is the meaning of life?",
+  "logit_bias": {
+    "1337": -100,
+    "42": 100
+  }
+}

llm/rust/triton-llm/tests/snapshots/openai_completions__valid_samples-2.snap

+---
+source: triton-llm/tests/openai_completions.rs
+description: "should have prompt, model, and max_tokens fields"
+expression: sample.request
+---
+{
+  "model": "gpt-3.5-turbo",
+  "prompt": "What is the meaning of life?",
+  "max_tokens": 10
+}

llm/rust/triton-llm/tests/snapshots/openai_completions__valid_samples-3.snap

+---
+source: triton-llm/tests/openai_completions.rs
+description: "should have prompt, model, and temperature fields"
+expression: sample.request
+---
+{
+  "model": "gpt-3.5-turbo",
+  "prompt": "What is the meaning of life?",
+  "temperature": 0.0
+}

llm/rust/triton-llm/tests/snapshots/openai_completions__valid_samples-4.snap

+---
+source: triton-llm/tests/openai_completions.rs
+description: "should have prompt, model, and top_p fields"
+expression: sample.request
+---
+{
+  "model": "gpt-3.5-turbo",
+  "prompt": "What is the meaning of life?",
+  "top_p": 0.0
+}

llm/rust/triton-llm/tests/snapshots/openai_completions__valid_samples-5.snap

+---
+source: triton-llm/tests/openai_completions.rs
+description: "should have prompt, model, and frequency_penalty fields"
+expression: sample.request
+---
+{
+  "model": "gpt-3.5-turbo",
+  "prompt": "What is the meaning of life?",
+  "frequency_penalty": -2.0
+}

llm/rust/triton-llm/tests/snapshots/openai_completions__valid_samples-6.snap

+---
+source: triton-llm/tests/openai_completions.rs
+description: "should have prompt, model, and presence_penalty fields"
+expression: sample.request
+---
+{
+  "model": "gpt-3.5-turbo",
+  "prompt": "What is the meaning of life?",
+  "presence_penalty": -2.0
+}

llm/rust/triton-llm/tests/snapshots/openai_completions__valid_samples-7.snap

+---
+source: triton-llm/tests/openai_completions.rs
+description: "should have prompt, model, and stop fields"
+expression: sample.request
+---
+{
+  "model": "gpt-3.5-turbo",
+  "prompt": "What is the meaning of life?",
+  "stop": [
+    "\n"
+  ]
+}