Skip to content

GitLab

Unverified Commit 9d9a1d9b authored by Anant Sharma's avatar Anant Sharma Committed by GitHub
Browse files

chore: update dynamo and nixl versions for 0.3.0 (#1240)

parent c9eb6a83
Hide whitespace changes
Inline Side-by-side
Showing with 79 additions and 54 deletions
Cargo.lock
View file @ 9d9a1d9b
...@@ -814,9 +814,9 @@ dependencies = [ ...@@ -814,9 +814,9 @@ dependencies = [
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.20" version = "1.2.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a" checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7"
dependencies = [ dependencies = [
"jobserver", "jobserver",
"libc", "libc",
...@@ -1555,7 +1555,7 @@ dependencies = [ ...@@ -1555,7 +1555,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-engine-llamacpp" name = "dynamo-engine-llamacpp"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"async-stream", "async-stream",
"dynamo-llm", "dynamo-llm",
...@@ -1567,7 +1567,7 @@ dependencies = [ ...@@ -1567,7 +1567,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-engine-mistralrs" name = "dynamo-engine-mistralrs"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-openai", "async-openai",
...@@ -1586,7 +1586,7 @@ dependencies = [ ...@@ -1586,7 +1586,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-llm" name = "dynamo-llm"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"akin", "akin",
"aligned-vec", "aligned-vec",
...@@ -1658,7 +1658,7 @@ dependencies = [ ...@@ -1658,7 +1658,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-run" name = "dynamo-run"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-openai", "async-openai",
...@@ -1686,7 +1686,7 @@ dependencies = [ ...@@ -1686,7 +1686,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-runtime" name = "dynamo-runtime"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"assert_matches", "assert_matches",
...@@ -1735,7 +1735,7 @@ dependencies = [ ...@@ -1735,7 +1735,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-tokens" name = "dynamo-tokens"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"bytemuck", "bytemuck",
"derive-getters", "derive-getters",
...@@ -2733,18 +2733,6 @@ dependencies = [ ...@@ -2733,18 +2733,6 @@ dependencies = [
"match_token", "match_token",
] ]
[[package]]
name = "http"
version = "0.2.1"
dependencies = [
"clap",
"dynamo-llm",
"dynamo-runtime",
"serde",
"serde_json",
"tokio",
]
[[package]] [[package]]
name = "http" name = "http"
version = "0.2.12" version = "0.2.12"
...@@ -2756,6 +2744,18 @@ dependencies = [ ...@@ -2756,6 +2744,18 @@ dependencies = [
"itoa", "itoa",
] ]
[[package]]
name = "http"
version = "0.3.0"
dependencies = [
"clap",
"dynamo-llm",
"dynamo-runtime",
"serde",
"serde_json",
"tokio",
]
[[package]] [[package]]
name = "http" name = "http"
version = "1.3.1" version = "1.3.1"
...@@ -3364,7 +3364,7 @@ checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" ...@@ -3364,7 +3364,7 @@ checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
[[package]] [[package]]
name = "libdynamo_llm" name = "libdynamo_llm"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-once-cell", "async-once-cell",
...@@ -3477,7 +3477,7 @@ dependencies = [ ...@@ -3477,7 +3477,7 @@ dependencies = [
[[package]] [[package]]
name = "llmctl" name = "llmctl"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"clap", "clap",
...@@ -3677,7 +3677,7 @@ dependencies = [ ...@@ -3677,7 +3677,7 @@ dependencies = [
[[package]] [[package]]
name = "metrics" name = "metrics"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"axum 0.6.20", "axum 0.6.20",
"clap", "clap",
...@@ -4084,13 +4084,14 @@ dependencies = [ ...@@ -4084,13 +4084,14 @@ dependencies = [
[[package]] [[package]]
name = "nixl-sys" name = "nixl-sys"
version = "0.2.1-rc.3" version = "0.3.0-rc.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfeec33e0229249e8688309a70c677f522446f9ac92105a85aad4a5ceef2dd2f" checksum = "6a1727039c315bf41aeadcdbea02522b8a3df69a77bd732996243502a5fb6f07"
dependencies = [ dependencies = [
"bindgen 0.71.1", "bindgen 0.71.1",
"cc", "cc",
"libc", "libc",
"os_info",
"pkg-config", "pkg-config",
"serde", "serde",
"thiserror 2.0.12", "thiserror 2.0.12",
...@@ -4387,6 +4388,17 @@ dependencies = [ ...@@ -4387,6 +4388,17 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "os_info"
version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fc863e2ca13dc2d5c34fb22ea4a588248ac14db929616ba65c45f21744b1e9"
dependencies = [
"log",
"serde",
"windows-sys 0.52.0",
]
[[package]] [[package]]
name = "overload" name = "overload"
version = "0.1.1" version = "0.1.1"
...@@ -5333,7 +5345,7 @@ dependencies = [ ...@@ -5333,7 +5345,7 @@ dependencies = [
[[package]] [[package]]
name = "router" name = "router"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"clap", "clap",
"dynamo-llm", "dynamo-llm",
......
Cargo.toml
View file @ 9d9a1d9b
...@@ -28,7 +28,7 @@ members = [ ...@@ -28,7 +28,7 @@ members = [
resolver = "3" resolver = "3"
[workspace.package] [workspace.package]
version = "0.2.1" version = "0.3.0"
edition = "2021" edition = "2021"
description = "Dynamo Inference Framework" description = "Dynamo Inference Framework"
authors = ["NVIDIA Inc. <sw-dl-dynamo@nvidia.com>"] authors = ["NVIDIA Inc. <sw-dl-dynamo@nvidia.com>"]
...@@ -39,9 +39,9 @@ keywords = ["llm", "genai", "inference", "nvidia", "distributed", "dynamo"] ...@@ -39,9 +39,9 @@ keywords = ["llm", "genai", "inference", "nvidia", "distributed", "dynamo"]
[workspace.dependencies] [workspace.dependencies]
# Local crates # Local crates
dynamo-runtime = { path = "lib/runtime", version = "0.2.1" } dynamo-runtime = { path = "lib/runtime", version = "0.3.0" }
dynamo-llm = { path = "lib/llm", version = "0.2.1" } dynamo-llm = { path = "lib/llm", version = "0.3.0" }
dynamo-tokens = { path = "lib/tokens", version = "0.2.1" } dynamo-tokens = { path = "lib/tokens", version = "0.3.0" }
# External dependencies # External dependencies
anyhow = { version = "1" } anyhow = { version = "1" }
......
container/Dockerfile.vllm
View file @ 9d9a1d9b
...@@ -169,7 +169,7 @@ RUN uv pip install /workspace/wheels/nixl/*.whl ...@@ -169,7 +169,7 @@ RUN uv pip install /workspace/wheels/nixl/*.whl
ARG VLLM_REF="0.8.4" ARG VLLM_REF="0.8.4"
ARG VLLM_PATCH="vllm_v${VLLM_REF}-dynamo-kv-disagg-patch.patch" ARG VLLM_PATCH="vllm_v${VLLM_REF}-dynamo-kv-disagg-patch.patch"
ARG VLLM_PATCHED_PACKAGE_NAME="ai_dynamo_vllm" ARG VLLM_PATCHED_PACKAGE_NAME="ai_dynamo_vllm"
ARG VLLM_PATCHED_PACKAGE_VERSION="0.8.4.post1" ARG VLLM_PATCHED_PACKAGE_VERSION="0.8.4.post2"
ARG VLLM_MAX_JOBS=4 ARG VLLM_MAX_JOBS=4
RUN --mount=type=bind,source=./container/deps/,target=/tmp/deps \ RUN --mount=type=bind,source=./container/deps/,target=/tmp/deps \
--mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,target=/root/.cache/uv \
......
container/build.sh
View file @ 9d9a1d9b
...@@ -109,7 +109,7 @@ NONE_BASE_IMAGE_TAG="24.04" ...@@ -109,7 +109,7 @@ NONE_BASE_IMAGE_TAG="24.04"
SGLANG_BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base" SGLANG_BASE_IMAGE="nvcr.io/nvidia/cuda-dl-base"
SGLANG_BASE_IMAGE_TAG="25.01-cuda12.8-devel-ubuntu24.04" SGLANG_BASE_IMAGE_TAG="25.01-cuda12.8-devel-ubuntu24.04"
NIXL_COMMIT=78695c2900cd7fff506764377386592dfc98e87e NIXL_COMMIT=f531404be4866d85ed618b3baf4008c636798d63
NIXL_REPO=ai-dynamo/nixl.git NIXL_REPO=ai-dynamo/nixl.git
NO_CACHE="" NO_CACHE=""
......
docs/support_matrix.md
View file @ 9d9a1d9b
...@@ -62,21 +62,21 @@ For **Linux**, the **ARM64** support is experimental and may have limitations. W ...@@ -62,21 +62,21 @@ For **Linux**, the **ARM64** support is experimental and may have limitations. W
### Runtime Dependency ### Runtime Dependency
| **Python Package** | **Version** | glibc version | CUDA Version | | **Python Package** | **Version** | glibc version | CUDA Version |
|--------------------|---------------|----------------------|--------------| |--------------------|---------------|----------------------|--------------|
| ai-dynamo | 0.2.1 | >=2.28 | | | ai-dynamo | 0.3.0 | >=2.28 | |
| ai-dynamo-runtime | 0.2.1 | >=2.28 | | | ai-dynamo-runtime | 0.3.0 | >=2.28 | |
| ai-dynamo-vllm | 0.8.4.post1* | >=2.28 (recommended) | | | ai-dynamo-vllm | 0.8.4.post2* | >=2.28 (recommended) | |
| NIXL | 0.2.1 | >=2.27 | >=11.8 | | NIXL | 0.3.0 | >=2.27 | >=11.8 |
### Build Dependency ### Build Dependency
| **Build Dependency** | **Version** | | **Build Dependency** | **Version** |
|----------------------|-------------| |----------------------|-------------|
| **Base Container** | [25.03](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda-dl-base/tags) | | **Base Container** | [25.03](https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda-dl-base/tags) |
| **ai-dynamo-vllm** |0.8.4.post1* | | **ai-dynamo-vllm** |0.8.4.post2* |
| **TensorRT-LLM** | 0.19.0** | | **TensorRT-LLM** | 0.19.0** |
| **NIXL** | 0.2.1 | | **NIXL** | 0.3.0 |
> **Note**: > **Note**:
> - *ai-dynamo-vllm v0.8.4.post1 is a customized patch of v0.8.4 from vLLM. > - *ai-dynamo-vllm v0.8.4.post2 is a customized patch of v0.8.4 from vLLM.
> - **Specific versions of TensorRT-LLM supported by Dynamo are subject to change. > - **Specific versions of TensorRT-LLM supported by Dynamo are subject to change.
......
lib/bindings/python/Cargo.lock
View file @ 9d9a1d9b
...@@ -577,9 +577,9 @@ dependencies = [ ...@@ -577,9 +577,9 @@ dependencies = [
[[package]] [[package]]
name = "cc" name = "cc"
version = "1.2.20" version = "1.2.24"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a" checksum = "16595d3be041c03b09d08d0858631facccee9221e579704070e6e9e4915d3bc7"
dependencies = [ dependencies = [
"jobserver", "jobserver",
"libc", "libc",
...@@ -1092,7 +1092,7 @@ dependencies = [ ...@@ -1092,7 +1092,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-llm" name = "dynamo-llm"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"akin", "akin",
"anyhow", "anyhow",
...@@ -1155,7 +1155,7 @@ dependencies = [ ...@@ -1155,7 +1155,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-py3" name = "dynamo-py3"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-openai", "async-openai",
...@@ -1180,7 +1180,7 @@ dependencies = [ ...@@ -1180,7 +1180,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-runtime" name = "dynamo-runtime"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-nats", "async-nats",
...@@ -2810,13 +2810,14 @@ dependencies = [ ...@@ -2810,13 +2810,14 @@ dependencies = [
[[package]] [[package]]
name = "nixl-sys" name = "nixl-sys"
version = "0.2.1" version = "0.3.0-rc.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84bf333c75733cad60b29873d84168f841c6bd5207ae9dfbda7490a99c1ebe94" checksum = "6a1727039c315bf41aeadcdbea02522b8a3df69a77bd732996243502a5fb6f07"
dependencies = [ dependencies = [
"bindgen", "bindgen",
"cc", "cc",
"libc", "libc",
"os_info",
"pkg-config", "pkg-config",
"serde", "serde",
"thiserror 2.0.12", "thiserror 2.0.12",
...@@ -3065,6 +3066,17 @@ version = "0.2.0" ...@@ -3065,6 +3066,17 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "os_info"
version = "3.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fc863e2ca13dc2d5c34fb22ea4a588248ac14db929616ba65c45f21744b1e9"
dependencies = [
"log",
"serde",
"windows-sys 0.52.0",
]
[[package]] [[package]]
name = "overload" name = "overload"
version = "0.1.1" version = "0.1.1"
......
lib/bindings/python/Cargo.toml
View file @ 9d9a1d9b
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
[package] [package]
name = "dynamo-py3" name = "dynamo-py3"
version = "0.2.1" version = "0.3.0"
edition = "2021" edition = "2021"
authors = ["NVIDIA"] authors = ["NVIDIA"]
license = "Apache-2.0" license = "Apache-2.0"
...@@ -75,3 +75,4 @@ pyo3-async-runtimes = { version = "0.23.0", default-features = false, features = ...@@ -75,3 +75,4 @@ pyo3-async-runtimes = { version = "0.23.0", default-features = false, features =
pythonize = "0.23" pythonize = "0.23"
dlpark = { version = "0.5", features = ["pyo3", "half"], optional = true } dlpark = { version = "0.5", features = ["pyo3", "half"], optional = true }
lib/llm/Cargo.toml
View file @ 9d9a1d9b
...@@ -81,7 +81,7 @@ regex = "1" ...@@ -81,7 +81,7 @@ regex = "1"
rayon = "1" rayon = "1"
# block_manager # block_manager
nixl-sys = { version = "0.2.1-rc.3", optional = true } nixl-sys = { version = "0.3.0-rc.2", optional = true }
cudarc = { version = "0.16.2", features = ["cuda-12020"], optional = true } cudarc = { version = "0.16.2", features = ["cuda-12020"], optional = true }
ndarray = { version = "0.16", optional = true } ndarray = { version = "0.16", optional = true }
nix = { version = "0.26", optional = true } nix = { version = "0.26", optional = true }
......
lib/runtime/examples/Cargo.lock
View file @ 9d9a1d9b
...@@ -623,7 +623,7 @@ dependencies = [ ...@@ -623,7 +623,7 @@ dependencies = [
[[package]] [[package]]
name = "dynamo-runtime" name = "dynamo-runtime"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-nats", "async-nats",
...@@ -994,7 +994,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" ...@@ -994,7 +994,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]] [[package]]
name = "hello_world" name = "hello_world"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"dynamo-runtime", "dynamo-runtime",
] ]
...@@ -2306,7 +2306,7 @@ dependencies = [ ...@@ -2306,7 +2306,7 @@ dependencies = [
[[package]] [[package]]
name = "service_metrics" name = "service_metrics"
version = "0.2.1" version = "0.3.0"
dependencies = [ dependencies = [
"dynamo-runtime", "dynamo-runtime",
"futures", "futures",
......
lib/runtime/examples/Cargo.toml
View file @ 9d9a1d9b
...@@ -21,7 +21,7 @@ members = [ ...@@ -21,7 +21,7 @@ members = [
resolver = "3" resolver = "3"
[workspace.package] [workspace.package]
version = "0.2.1" version = "0.3.0"
edition = "2021" edition = "2021"
authors = ["NVIDIA"] authors = ["NVIDIA"]
license = "Apache-2.0" license = "Apache-2.0"
......
pyproject.toml
View file @ 9d9a1d9b
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[project] [project]
name = "ai-dynamo" name = "ai-dynamo"
version = "0.2.1" version = "0.3.0"
description = "Distributed Inference Framework" description = "Distributed Inference Framework"
readme = "README.md" readme = "README.md"
authors = [ authors = [
...@@ -29,7 +29,7 @@ dependencies = [ ...@@ -29,7 +29,7 @@ dependencies = [
"bentoml==1.4.8", "bentoml==1.4.8",
"types-psutil==7.0.0.20250218", "types-psutil==7.0.0.20250218",
"kubernetes==32.0.1", "kubernetes==32.0.1",
"ai-dynamo-runtime==0.2.1", "ai-dynamo-runtime==0.3.0",
"fastapi==0.115.6", "fastapi==0.115.6",
"distro", "distro",
"typer", "typer",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment