fix: add VLLM_KV_CAPI_PATH to vllm dockerfile to make kv routing working (#832)

f5e8488c · Ziqi Fan · GitHub · 1de737fe · f5e8488c
Unverified Commit f5e8488c authored Apr 25, 2025 by Ziqi Fan Committed by GitHub Apr 25, 2025
Show whitespace changes
Inline Side-by-side

Showing with 2 additions and 0 deletions

container/Dockerfile.vllm container/Dockerfile.vllm +2 -0

No files found.
--- a/container/Dockerfile.vllm
+++ b/container/Dockerfile.vllm
@@ -414,6 +414,8 @@ RUN --mount=type=bind,source=./container/launch_message.txt,target=/workspace/la
    sed '/^#\s/d' /workspace/launch_message.txt > ~/.launch_screen && \
    echo "cat ~/.launch_screen" >> ~/.bashrc

+# Tell vllm to use the Dynamo LLM C API for KV Cache Routing
+ENV VLLM_KV_CAPI_PATH=$CARGO_TARGET_DIR/release/libdynamo_llm_capi.so

 ##########################################
 ########## Perf Analyzer Image ###########