[Bugfix] Set enable_prefix_caching=True in prefix caching example (#3703)

c0935c96 · Woosuk Kwon · GitHub · cb40b3ab · c0935c96
Unverified Commit c0935c96 authored Mar 28, 2024 by Woosuk Kwon Committed by GitHub Mar 28, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

examples/offline_inference_with_prefix.py examples/offline_inference_with_prefix.py +1 -1

No files found.
--- a/examples/offline_inference_with_prefix.py
+++ b/examples/offline_inference_with_prefix.py
@@ -22,7 +22,7 @@ prompts = [
 sampling_params = SamplingParams(temperature=0.0)

 # Create an LLM.
-llm = LLM(model="facebook/opt-125m")
+llm = LLM(model="facebook/opt-125m", enable_prefix_caching=True)

 generating_prompts = [prefix + prompt for prompt in prompts]