feat: Support a small runtime container (#167)

.github/workflows/pre-merge-python.yml

@@ -69,7 +69,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.CI_TOKEN }}
         run: |
-          ./container/build.sh --tag ${{ steps.define_image_tag.outputs.image_tag }} --framework ${{ matrix.framework }} ${{ steps.which_caches.outputs.cache_from_location }} ${{ steps.which_caches.outputs.cache_to_location }}
+          ./container/build.sh --tag ${{ steps.define_image_tag.outputs.image_tag }} --target dev --framework ${{ matrix.framework }} ${{ steps.which_caches.outputs.cache_from_location }} ${{ steps.which_caches.outputs.cache_to_location }}
       - name: Run pytest
         env:
           PYTEST_MARKS: "pre_merge or mypy"

container/Dockerfile.vllm

@@ -4,6 +4,9 @@
 ARG BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base"
 ARG BASE_IMAGE_TAG="25.01-cuda12.8-devel-ubuntu24.04"
 
+ARG RUNTIME_IMAGE="nvcr.io/nvidia/cuda"
+ARG RUNTIME_IMAGE_TAG="12.8.1-runtime-ubuntu24.04"
+
 FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} AS nixl_base
 WORKDIR /opt/nixl
 # Add a cache hint that only changes when the nixl commit changes
@@ -269,5 +272,32 @@ ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
 
 CMD []
 
-### TODO Lean Runtime Image Stage ###
+########################################
+########## RUNTIME CONTAINER ###########
+########################################
+
+FROM ${RUNTIME_IMAGE}:${RUNTIME_IMAGE_TAG} AS runtime
+
+WORKDIR /workspace
+ENV VIRTUAL_ENV=/opt/dynamo/venv
+
+RUN rm /bin/sh && ln -s /bin/bash /bin/sh
+
+# Setup the python environment
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+RUN uv venv $VIRTUAL_ENV --python 3.12 && \
+    ln -s $(find / -name libpython3.12.so*) /usr/lib && \
+    echo "source $VIRTUAL_ENV/bin/activate" >> ~/.bashrc
+
+# Install the wheels
+COPY --from=dev /workspace/dist/*.whl whls/
+RUN uv pip install $(find whls -name ai_dynamo_runtime-*.whl) && \
+    uv pip install $(find whls -name ai_dynamo-*.whl) && \
+    uv pip install $(find whls -name vllm*cp312*.whl) && \
+    rm -r whls
+
+# Tell vllm to use the Dynamo LLM C API for KV Cache Routing
+ENV VLLM_KV_CAPI_PATH="/opt/dynamo/bindings/lib/libdynamo_llm_capi.so"
 
+# dynamo run in=text out=mistralrs Qwen/Qwen2.5-3B-Instruct
+CMD []
\ No newline at end of file

container/build.sh

@@ -229,6 +229,8 @@ get_options() {
 
     if [ ! -z "$TARGET" ]; then
         TARGET_STR="--target ${TARGET}"
+    else
+        TARGET_STR="--target runtime"
     fi
 }
 
@@ -355,4 +357,4 @@ if [ -z "$RUN_PREFIX" ]; then
     set -x
 fi
 
-{ set +x; } 2>/dev/null
+{ set +x; } 2>/dev/null
\ No newline at end of file