Skip to content

GitLab

"lib/llm/src/vscode:/vscode.git/clone" did not exist on "124ecd98045ada1ef22e5cd442dae642a25b83ac"
Unverified Commit bbe82f18 authored by Graham King's avatar Graham King Committed by GitHub
Browse files

chore: Remove dynamo-run and mistral-rs engine (#6203) 


Signed-off-by: default avatarGraham King <grahamk@nvidia.com>
parent 2c747d64
Show whitespace changes
Inline Side-by-side
Showing with 80 additions and 2486 deletions
.devcontainer/README.md
View file @ bbe82f18
......@@ -211,7 +211,7 @@ If `post-create.sh` fails, you can try to debug or [submit](https://github.com/a
### Building Rust Code
If you make changes to Rust code and want to compile, use [cargo build](https://doc.rust-lang.org/cargo/commands/cargo-build.html). This will update Rust binaries such as dynamo-run.
If you make changes to Rust code and want to compile, use [cargo build](https://doc.rust-lang.org/cargo/commands/cargo-build.html). This will update Rust binaries.
```bash
cd /workspace && cargo build --locked --profile dev
......
.devcontainer/devcontainer.json.j2
View file @ bbe82f18
......@@ -49,7 +49,6 @@
"lib/llm/Cargo.toml",
"lib/tokens/Cargo.toml",
"lib/bindings/python/Cargo.toml",
"launch/dynamo-run/Cargo.toml"
],
"files.trimTrailingWhitespace": true,
"files.insertFinalNewline": true
......
.devcontainer/sglang/devcontainer.json
View file @ bbe82f18
......@@ -4,7 +4,7 @@
// To make changes, edit the .j2 template and run gen_devcontainer_json.py
"$schema": "https://raw.githubusercontent.com/devcontainers/spec/main/schemas/devContainer.schema.json",
"copyright": [
"SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.",
"SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.",
"SPDX-License-Identifier: Apache-2.0"
],
"name": "Dynamo SGLANG Dev Container",
......@@ -52,7 +52,6 @@
"lib/llm/Cargo.toml",
"lib/tokens/Cargo.toml",
"lib/bindings/python/Cargo.toml",
"launch/dynamo-run/Cargo.toml"
],
"files.trimTrailingWhitespace": true,
"files.insertFinalNewline": true
......
.devcontainer/trtllm/devcontainer.json
View file @ bbe82f18
......@@ -4,7 +4,7 @@
// To make changes, edit the .j2 template and run gen_devcontainer_json.py
"$schema": "https://raw.githubusercontent.com/devcontainers/spec/main/schemas/devContainer.schema.json",
"copyright": [
"SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.",
"SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.",
"SPDX-License-Identifier: Apache-2.0"
],
"name": "Dynamo TRTLLM Dev Container",
......@@ -52,7 +52,6 @@
"lib/llm/Cargo.toml",
"lib/tokens/Cargo.toml",
"lib/bindings/python/Cargo.toml",
"launch/dynamo-run/Cargo.toml"
],
"files.trimTrailingWhitespace": true,
"files.insertFinalNewline": true
......
.devcontainer/vllm/devcontainer.json
View file @ bbe82f18
......@@ -4,7 +4,7 @@
// To make changes, edit the .j2 template and run gen_devcontainer_json.py
"$schema": "https://raw.githubusercontent.com/devcontainers/spec/main/schemas/devContainer.schema.json",
"copyright": [
"SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.",
"SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.",
"SPDX-License-Identifier: Apache-2.0"
],
"name": "Dynamo VLLM Dev Container",
......@@ -52,7 +52,6 @@
"lib/llm/Cargo.toml",
"lib/tokens/Cargo.toml",
"lib/bindings/python/Cargo.toml",
"launch/dynamo-run/Cargo.toml"
],
"files.trimTrailingWhitespace": true,
"files.insertFinalNewline": true
......
.github/filters.yaml
View file @ bbe82f18
......@@ -44,7 +44,6 @@ ignore:
- '.clang-format'
- 'dynamo.code-workspace'
- 'deploy/*'
- 'launch/**'
- '.github/scripts/**'
- '.github/ISSUE_TEMPLATE/**'
- '.github/pull_request_template.md'
......
.github/workflows/pre-merge.yml
View file @ bbe82f18
......@@ -68,7 +68,7 @@ jobs:
runs-on:
group: Fastchecker
strategy:
matrix: { dir: ['.', 'lib/bindings/python', 'lib/runtime/examples', 'launch/dynamo-run', 'lib/bindings/kvbm'] }
matrix: { dir: ['.', 'lib/bindings/python', 'lib/runtime/examples', 'lib/bindings/kvbm'] }
permissions:
contents: read
steps:
......@@ -124,7 +124,7 @@ jobs:
runs-on:
group: Fastchecker
strategy:
matrix: { dir: ['.', 'lib/bindings/python', 'lib/runtime/examples', 'launch/dynamo-run', 'lib/bindings/kvbm'] }
matrix: { dir: ['.', 'lib/bindings/python', 'lib/runtime/examples', 'lib/bindings/kvbm'] }
permissions:
contents: read
steps:
......
Cargo.lock
View file @ bbe82f18
......@@ -2,16 +2,6 @@
# It is not intended for manual editing.
version = 4
[[package]]
name = "Inflector"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
dependencies = [
"lazy_static",
"regex",
]
[[package]]
name = "addr2line"
version = "0.25.1"
......@@ -154,12 +144,6 @@ dependencies = [
"backtrace",
]
[[package]]
name = "apodize"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fca387cdc0a1f9c7a7c26556d584aa2d07fc529843082e4861003cde4ab914ed"
[[package]]
name = "approx"
version = "0.5.1"
......@@ -213,12 +197,6 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "as-any"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063"
[[package]]
name = "as-slice"
version = "0.2.1"
......@@ -732,27 +710,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "bindgen_cuda"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f8489af5b7d17a81bffe37e0f4d6e1e4de87c87329d05447f22c35d95a1227d"
dependencies = [
"glob",
"num_cpus",
"rayon",
]
[[package]]
name = "bindgen_cuda"
version = "0.1.7"
source = "git+https://github.com/guoqingbao/bindgen_cuda.git#19e33d0e55fec148f53aaed144de401ff1fd9a6a"
dependencies = [
"glob",
"num_cpus",
"rayon",
]
[[package]]
name = "bit-set"
version = "0.5.3"
......@@ -828,12 +785,6 @@ dependencies = [
"rayon-core",
]
[[package]]
name = "block"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"
[[package]]
name = "block-buffer"
version = "0.10.4"
......@@ -852,20 +803,6 @@ dependencies = [
"objc2",
]
[[package]]
name = "bm25"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cbd8ffdfb7b4c2ff038726178a780a94f90525ed0ad264c0afaa75dd8c18a64"
dependencies = [
"cached",
"deunicode",
"fxhash",
"rust-stemmers",
"stop-words",
"unicode-segmentation",
]
[[package]]
name = "bs58"
version = "0.5.1"
......@@ -956,39 +893,6 @@ dependencies = [
"serde",
]
[[package]]
name = "cached"
version = "0.56.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "801927ee168e17809ab8901d9f01f700cd7d8d6a6527997fee44e4b0327a253c"
dependencies = [
"ahash",
"cached_proc_macro",
"cached_proc_macro_types",
"hashbrown 0.15.5",
"once_cell",
"thiserror 2.0.17",
"web-time",
]
[[package]]
name = "cached_proc_macro"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9225bdcf4e4a9a4c08bf16607908eb2fbf746828d5e0b5e019726dbf6571f201"
dependencies = [
"darling 0.20.11",
"proc-macro2",
"quote",
"syn 2.0.111",
]
[[package]]
name = "cached_proc_macro_types"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0"
[[package]]
name = "candle-core"
version = "0.9.1"
......@@ -1002,79 +906,15 @@ dependencies = [
"num-traits",
"num_cpus",
"rand 0.9.2",
"rand_distr 0.5.1",
"rayon",
"safetensors 0.4.5",
"thiserror 1.0.69",
"ug 0.4.0",
"yoke 0.7.5",
"zip 1.1.4",
]
[[package]]
name = "candle-core"
version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=175926c9#175926c960f5aff6a543a06b3214471dd92785e4"
dependencies = [
"byteorder",
"candle-kernels",
"candle-metal-kernels",
"cudarc",
"float8",
"gemm 0.17.1",
"half 2.7.1",
"memmap2",
"metal 0.27.0",
"num-traits",
"num_cpus",
"rand 0.9.2",
"rand_distr 0.5.1",
"rand_distr",
"rayon",
"safetensors 0.6.2",
"safetensors",
"thiserror 1.0.69",
"ug 0.5.0",
"ug-cuda",
"ug-metal",
"ug",
"yoke 0.7.5",
"zip 1.1.4",
]
[[package]]
name = "candle-kernels"
version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=175926c9#175926c960f5aff6a543a06b3214471dd92785e4"
dependencies = [
"bindgen_cuda 0.1.5",
]
[[package]]
name = "candle-metal-kernels"
version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=175926c9#175926c960f5aff6a543a06b3214471dd92785e4"
dependencies = [
"half 2.7.1",
"metal 0.27.0",
"once_cell",
"thiserror 1.0.69",
"tracing",
]
[[package]]
name = "candle-nn"
version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=175926c9#175926c960f5aff6a543a06b3214471dd92785e4"
dependencies = [
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"candle-metal-kernels",
"half 2.7.1",
"metal 0.27.0",
"num-traits",
"rayon",
"safetensors 0.6.2",
"serde",
"thiserror 1.0.69",
]
[[package]]
name = "cast"
version = "0.3.0"
......@@ -1158,20 +998,6 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "cfgrammar"
version = "0.13.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fe45e18904af7af10e4312df7c97251e98af98c70f42f1f2587aecfcbee56bf"
dependencies = [
"indexmap 2.12.1",
"lazy_static",
"num-traits",
"regex",
"serde",
"vob",
]
[[package]]
name = "chrono"
version = "0.4.42"
......@@ -1254,8 +1080,7 @@ dependencies = [
"anstream",
"anstyle",
"clap_lex",
"strsim 0.11.1",
"terminal_size",
"strsim",
]
[[package]]
......@@ -1482,17 +1307,6 @@ version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
[[package]]
name = "core-graphics-types"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45390e6114f68f718cc7a830514a96f903cccd70d02a8f6d9f643ac4ba45afaf"
dependencies = [
"bitflags 1.3.2",
"core-foundation 0.9.4",
"libc",
]
[[package]]
name = "core2"
version = "0.4.0"
......@@ -1650,31 +1464,6 @@ version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crossterm"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67"
dependencies = [
"bitflags 1.3.2",
"crossterm_winapi",
"libc",
"mio 0.8.11",
"parking_lot",
"signal-hook",
"signal-hook-mio",
"winapi 0.3.9",
]
[[package]]
name = "crossterm_winapi"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b"
dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "crunchy"
version = "0.2.4"
......@@ -1691,29 +1480,6 @@ dependencies = [
"typenum",
]
[[package]]
name = "cssparser"
version = "0.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3"
dependencies = [
"cssparser-macros",
"dtoa-short",
"itoa",
"phf",
"smallvec",
]
[[package]]
name = "cssparser-macros"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
dependencies = [
"quote",
"syn 2.0.111",
]
[[package]]
name = "csv"
version = "1.4.0"
......@@ -1751,7 +1517,6 @@ version = "0.17.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf99ab37ee7072d64d906aa2dada9a3422f1d975cdf8c8055a573bc84897ed8"
dependencies = [
"half 2.7.1",
"libloading",
]
......@@ -1781,16 +1546,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "darling"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbffa8f8e38810422f320ca457a93cf1cd0056dc9c06c556b867558e0d471463"
dependencies = [
"darling_core 0.11.0",
"darling_macro 0.11.0",
]
[[package]]
name = "darling"
version = "0.20.11"
......@@ -1811,20 +1566,6 @@ dependencies = [
"darling_macro 0.21.3",
]
[[package]]
name = "darling_core"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06e172685d94b7b83800e3256a63261537b9d6129e10f21c8e13ddf9dba8c64d"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim 0.10.0",
"syn 1.0.109",
]
[[package]]
name = "darling_core"
version = "0.20.11"
......@@ -1835,7 +1576,7 @@ dependencies = [
"ident_case",
"proc-macro2",
"quote",
"strsim 0.11.1",
"strsim",
"syn 2.0.111",
]
......@@ -1849,21 +1590,10 @@ dependencies = [
"ident_case",
"proc-macro2",
"quote",
"strsim 0.11.1",
"strsim",
"syn 2.0.111",
]
[[package]]
name = "darling_macro"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0618ac802792cebd1918ac6042a6ea1eeab92db34b35656afaa577929820788"
dependencies = [
"darling_core 0.11.0",
"quote",
"syn 1.0.109",
]
[[package]]
name = "darling_macro"
version = "0.20.11"
......@@ -1966,17 +1696,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "derive-new"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2cdc8d50f426189eef89dac62fabfa0abb27d5cc008f25bf4156a0203325becc"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.111",
]
[[package]]
name = "derive_arbitrary"
version = "1.4.2"
......@@ -2019,17 +1738,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "derive_more"
version = "0.99.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.111",
]
[[package]]
name = "derive_more"
version = "1.0.0"
......@@ -2071,26 +1779,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "derivre"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "786c7c65c4ef0c7deb05de3005e01991612a8f09fe0844fc0969c68b90468ba8"
dependencies = [
"anyhow",
"bytemuck",
"bytemuck_derive",
"hashbrown 0.15.5",
"regex-syntax",
"strum",
]
[[package]]
name = "deunicode"
version = "1.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
[[package]]
name = "dialoguer"
version = "0.11.0"
......@@ -2175,27 +1863,6 @@ dependencies = [
"const-random",
]
[[package]]
name = "doctest-file"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562"
[[package]]
name = "dtoa"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04"
[[package]]
name = "dtoa-short"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
dependencies = [
"dtoa",
]
[[package]]
name = "dunce"
version = "1.0.5"
......@@ -2255,12 +1922,12 @@ dependencies = [
"tokio",
"tokio-stream",
"tokio-test",
"tokio-tungstenite 0.26.2",
"tokio-tungstenite",
"tokio-util",
"tracing",
"url",
"utoipa",
"uuid 1.18.1",
"uuid",
]
[[package]]
......@@ -2280,24 +1947,6 @@ dependencies = [
"anyhow",
]
[[package]]
name = "dynamo-engine-mistralrs"
version = "0.9.0"
dependencies = [
"anyhow",
"async-stream",
"async-trait",
"dynamo-async-openai",
"dynamo-llm",
"dynamo-runtime",
"either",
"indexmap 2.12.1",
"mistralrs",
"serde_json",
"tokio",
"tracing",
]
[[package]]
name = "dynamo-kv-router"
version = "0.9.0"
......@@ -2324,7 +1973,7 @@ dependencies = [
"tokio",
"tokio-util",
"tracing",
"uuid 1.18.1",
"uuid",
"xxhash-rust",
]
......@@ -2352,7 +2001,7 @@ dependencies = [
"bs62",
"bytemuck",
"bytes",
"candle-core 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"candle-core",
"chrono",
"clap 4.5.53",
"criterion 0.3.6",
......@@ -2425,8 +2074,8 @@ dependencies = [
"tokio-rayon",
"tokio-stream",
"tokio-util",
"toktrie 1.4.0",
"toktrie_hf_tokenizers 1.4.0",
"toktrie",
"toktrie_hf_tokenizers",
"tonic 0.13.1",
"tonic-build 0.13.1",
"tower 0.5.2",
......@@ -2436,7 +2085,7 @@ dependencies = [
"url",
"utoipa",
"utoipa-swagger-ui",
"uuid 1.18.1",
"uuid",
"validator",
"video-rs",
"xxhash-rust",
......@@ -2481,7 +2130,7 @@ dependencies = [
"tokio",
"tokio-util",
"tracing",
"uuid 1.18.1",
"uuid",
"validator",
]
......@@ -2501,39 +2150,11 @@ dependencies = [
"serde_json",
"tokio",
"tracing",
"uuid 1.18.1",
"uuid",
]
[[package]]
name = "dynamo-run"
version = "0.9.0"
dependencies = [
"anyhow",
"async-stream",
"async-trait",
"clap 4.5.53",
"dynamo-async-openai",
"dynamo-engine-mistralrs",
"dynamo-llm",
"dynamo-runtime",
"either",
"futures",
"futures-util",
"libc",
"regex",
"serde",
"serde_json",
"tempfile",
"tokio",
"tokio-util",
"tracing",
"tracing-subscriber",
"uuid 1.18.1",
"vergen-gitcl",
]
[[package]]
name = "dynamo-runtime"
name = "dynamo-runtime"
version = "0.9.0"
dependencies = [
"anyhow",
......@@ -2607,7 +2228,7 @@ dependencies = [
"tracing-opentelemetry",
"tracing-subscriber",
"url",
"uuid 1.18.1",
"uuid",
"validator",
"xxhash-rust",
"zmq",
......@@ -2623,7 +2244,7 @@ dependencies = [
"derive-getters",
"serde",
"thiserror 2.0.17",
"uuid 1.18.1",
"uuid",
"xxhash-rust",
]
......@@ -2661,12 +2282,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "ego-tree"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8"
[[package]]
name = "either"
version = "1.15.0"
......@@ -2691,12 +2306,6 @@ dependencies = [
"cfg-if 1.0.4",
]
[[package]]
name = "endian-type"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
[[package]]
name = "enum-as-inner"
version = "0.6.1"
......@@ -2873,12 +2482,6 @@ dependencies = [
"zune-inflate",
]
[[package]]
name = "extended"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
[[package]]
name = "fancy-regex"
version = "0.11.0"
......@@ -3032,17 +2635,6 @@ dependencies = [
"miniz_oxide",
]
[[package]]
name = "float8"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4203231de188ebbdfb85c11f3c20ca2b063945710de04e7b59268731e728b462"
dependencies = [
"cudarc",
"half 2.7.1",
"num-traits",
]
[[package]]
name = "flume"
version = "0.12.0"
......@@ -3067,33 +2659,6 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foreign-types"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
dependencies = [
"foreign-types-macros",
"foreign-types-shared",
]
[[package]]
name = "foreign-types-macros"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.111",
]
[[package]]
name = "foreign-types-shared"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b"
[[package]]
name = "form_urlencoded"
version = "1.2.2"
......@@ -3154,16 +2719,6 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
[[package]]
name = "futf"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
dependencies = [
"mac",
"new_debug_unreachable",
]
[[package]]
name = "futures"
version = "0.3.31"
......@@ -3259,15 +2814,6 @@ dependencies = [
"slab",
]
[[package]]
name = "fxhash"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
dependencies = [
"byteorder",
]
[[package]]
name = "galil-seiferas"
version = "0.1.5"
......@@ -3663,7 +3209,7 @@ dependencies = [
"crunchy",
"num-traits",
"rand 0.9.2",
"rand_distr 0.5.1",
"rand_distr",
"zerocopy",
]
......@@ -3797,48 +3343,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "hound"
version = "3.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
[[package]]
name = "html2text"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1637acec3b965bab873352189d887b12c87b4f8d7571f4d185e796be5654ad8"
dependencies = [
"html5ever 0.31.0",
"tendril",
"thiserror 2.0.17",
"unicode-width 0.2.2",
]
[[package]]
name = "html5ever"
version = "0.29.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c"
dependencies = [
"log",
"mac",
"markup5ever 0.14.1",
"match_token",
]
[[package]]
name = "html5ever"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "953cbbe631aae7fc0a112702ad5d3aaf09da38beaf45ea84610d6e1c358f569c"
dependencies = [
"log",
"mac",
"markup5ever 0.16.2",
"match_token",
]
[[package]]
name = "http"
version = "0.2.12"
......@@ -4029,7 +3533,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.62.2",
"windows-core",
]
[[package]]
......@@ -4222,7 +3726,6 @@ dependencies = [
"console 0.15.11",
"number_prefix",
"portable-atomic",
"rayon",
"unicode-width 0.2.2",
"web-time",
]
......@@ -4316,19 +3819,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "interprocess"
version = "2.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d941b405bd2322993887859a8ee6ac9134945a24ec5ec763a8a962fc64dfec2d"
dependencies = [
"doctest-file",
"libc",
"recvmsg",
"widestring",
"windows-sys 0.52.0",
]
[[package]]
name = "iovec"
version = "0.1.4"
......@@ -4569,7 +4059,7 @@ dependencies = [
"serde_json",
"time",
"url",
"uuid 1.18.1",
"uuid",
]
[[package]]
......@@ -4775,7 +4265,7 @@ dependencies = [
"tokio-stream",
"tracing",
"tracing-subscriber",
"uuid 1.18.1",
"uuid",
]
[[package]]
......@@ -4836,20 +4326,6 @@ version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
[[package]]
name = "llguidance"
version = "1.0.0"
source = "git+https://github.com/guidance-ai/llguidance.git?rev=c432092#c432092d37b8ccd1afeeff3e7f9c9a29aae0a87e"
dependencies = [
"anyhow",
"derivre",
"indexmap 2.12.1",
"regex-syntax",
"serde",
"serde_json",
"toktrie 1.0.0",
]
[[package]]
name = "local-ip-address"
version = "0.6.5"
......@@ -4886,19 +4362,6 @@ dependencies = [
"imgref",
]
[[package]]
name = "lrtable"
version = "0.13.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc36d15214ca997a5097845be1f932b7ee6125c36f5c5e55f6c49e027ddeb6de"
dependencies = [
"cfgrammar",
"fnv",
"num-traits",
"sparsevec",
"vob",
]
[[package]]
name = "lru"
version = "0.12.5"
......@@ -4914,12 +4377,6 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "mac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "macro_rules_attribute"
version = "0.2.2"
......@@ -4994,51 +4451,6 @@ dependencies = [
"malachite-nz",
]
[[package]]
name = "malloc_buf"
version = "0.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb"
dependencies = [
"libc",
]
[[package]]
name = "markup5ever"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18"
dependencies = [
"log",
"phf",
"phf_codegen",
"string_cache",
"string_cache_codegen",
"tendril",
]
[[package]]
name = "markup5ever"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e4cd8c02f18a011991a039855480c64d74291c5792fcc160d55d77dc4de4a39"
dependencies = [
"log",
"tendril",
"web_atoms",
]
[[package]]
name = "match_token"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.111",
]
[[package]]
name = "matchers"
version = "0.2.0"
......@@ -5130,36 +4542,6 @@ dependencies = [
"autocfg",
]
[[package]]
name = "metal"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c43f73953f8cbe511f021b58f18c3ce1c3d1ae13fe953293e13345bf83217f25"
dependencies = [
"bitflags 2.10.0",
"block",
"core-graphics-types",
"foreign-types",
"log",
"objc",
"paste",
]
[[package]]
name = "metal"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ecfd3296f8c56b7c1f6fbac3c71cefa9d78ce009850c45000015f206dc7fa21"
dependencies = [
"bitflags 2.10.0",
"block",
"core-graphics-types",
"foreign-types",
"log",
"objc",
"paste",
]
[[package]]
name = "mime"
version = "0.3.17"
......@@ -5185,7 +4567,6 @@ dependencies = [
"memo-map",
"self_cell",
"serde",
"serde_json",
]
[[package]]
......@@ -5279,264 +4660,63 @@ dependencies = [
]
[[package]]
name = "mistralrs"
version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
name = "mockito"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e0603425789b4a70fcc4ac4f5a46a566c116ee3e2a6b768dc623f7719c611de"
dependencies = [
"anyhow",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"candle-nn",
"clap 4.5.53",
"either",
"futures",
"image",
"indexmap 2.12.1",
"mistralrs-core",
"assert-json-diff",
"bytes",
"colored",
"futures-core",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.8.1",
"hyper-util",
"log",
"pin-project-lite",
"rand 0.9.2",
"reqwest 0.12.24",
"serde",
"regex",
"serde_json",
"serde_urlencoded",
"similar",
"tokio",
"tracing",
"tracing-subscriber",
"walkdir",
]
[[package]]
name = "mistralrs-audio"
version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
name = "modelexpress-client"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9462ec6cd8b3d6beaa262ad0907a8ba297c7e3a220c11e4159742d8c275588eb"
dependencies = [
"anyhow",
"apodize",
"hound",
"symphonia",
"clap 4.5.53",
"colored",
"futures",
"modelexpress-common",
"prost 0.13.5",
"serde",
"serde_json",
"thiserror 2.0.17",
"tokio",
"tonic 0.13.1",
"tracing",
"tracing-subscriber",
"uuid",
]
[[package]]
name = "mistralrs-core"
version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
name = "modelexpress-common"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd921793901b2a39105accab89a953d96661cc0eebfecfda307bec266edafbb"
dependencies = [
"ahash",
"akin",
"anyhow",
"apodize",
"as-any",
"async-trait",
"base64 0.22.1",
"bindgen_cuda 0.1.7",
"bm25",
"bytemuck",
"bytemuck_derive",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"candle-nn",
"cfgrammar",
"chrono",
"clap 4.5.53",
"csv",
"derive-new",
"derive_more 2.0.1",
"dirs",
"either",
"float8",
"futures",
"galil-seiferas",
"half 2.7.1",
"hashbrown 0.15.5",
"hf-hub",
"hound",
"html2text",
"http 1.4.0",
"image",
"indexmap 2.12.1",
"indicatif 0.17.11",
"interprocess",
"itertools 0.14.0",
"libc",
"llguidance",
"lrtable",
"metal 0.27.0",
"minijinja",
"minijinja-contrib",
"mistralrs-audio",
"mistralrs-mcp",
"mistralrs-paged-attn",
"mistralrs-quant",
"mistralrs-vision",
"num-traits",
"objc",
"ordered-float 5.1.0",
"parking_lot",
"radix_trie",
"rand 0.9.2",
"rand_distr 0.5.1",
"rand_isaac",
"rayon",
"regex",
"regex-automata",
"reqwest 0.12.24",
"rubato",
"rust-mcp-schema",
"rustc-hash 2.1.1",
"rustfft",
"safetensors 0.6.2",
"schemars 0.8.22",
"scraper",
"serde",
"serde-big-array",
"serde_json",
"serde_plain",
"serde_yaml",
"statrs",
"strum",
"symphonia",
"sysinfo",
"thiserror 2.0.17",
"tokenizers",
"tokio",
"tokio-rayon",
"tokio-tungstenite 0.24.0",
"toktrie_hf_tokenizers 1.0.0",
"toml 0.8.23",
"tqdm",
"tracing",
"tracing-subscriber",
"urlencoding",
"uuid 1.18.1",
"variantly",
"vob",
]
[[package]]
name = "mistralrs-mcp"
version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [
"anyhow",
"async-trait",
"futures-util",
"http 1.4.0",
"reqwest 0.12.24",
"rust-mcp-schema",
"serde",
"serde_json",
"tokio",
"tokio-tungstenite 0.24.0",
"tracing",
"utoipa",
"uuid 1.18.1",
]
[[package]]
name = "mistralrs-paged-attn"
version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [
"anyhow",
"bindgen_cuda 0.1.7",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"float8",
"half 2.7.1",
"metal 0.27.0",
"thiserror 2.0.17",
]
[[package]]
name = "mistralrs-quant"
version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [
"bindgen_cuda 0.1.7",
"byteorder",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"candle-nn",
"float8",
"half 2.7.1",
"hf-hub",
"lazy_static",
"memmap2",
"metal 0.27.0",
"paste",
"rayon",
"regex",
"safetensors 0.6.2",
"serde",
"serde_json",
"thiserror 2.0.17",
"tokio",
"tracing",
"yoke 0.7.5",
]
[[package]]
name = "mistralrs-vision"
version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"image",
"rayon",
]
[[package]]
name = "mockito"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e0603425789b4a70fcc4ac4f5a46a566c116ee3e2a6b768dc623f7719c611de"
dependencies = [
"assert-json-diff",
"bytes",
"colored",
"futures-core",
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.8.1",
"hyper-util",
"log",
"pin-project-lite",
"rand 0.9.2",
"regex",
"serde_json",
"serde_urlencoded",
"similar",
"tokio",
]
[[package]]
name = "modelexpress-client"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9462ec6cd8b3d6beaa262ad0907a8ba297c7e3a220c11e4159742d8c275588eb"
dependencies = [
"anyhow",
"clap 4.5.53",
"colored",
"futures",
"modelexpress-common",
"prost 0.13.5",
"serde",
"serde_json",
"thiserror 2.0.17",
"tokio",
"tonic 0.13.1",
"tracing",
"tracing-subscriber",
"uuid 1.18.1",
]
[[package]]
name = "modelexpress-common"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd921793901b2a39105accab89a953d96661cc0eebfecfda307bec266edafbb"
dependencies = [
"anyhow",
"async-trait",
"chrono",
"clap 4.5.53",
"config",
"config",
"hf-hub",
"jiff",
"prost 0.13.5",
......@@ -5588,23 +4768,6 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
[[package]]
name = "nalgebra"
version = "0.33.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b"
dependencies = [
"approx",
"matrixmultiply",
"num-complex",
"num-rational",
"num-traits",
"rand 0.8.5",
"rand_distr 0.4.3",
"simba",
"typenum",
]
[[package]]
name = "ndarray"
version = "0.16.1"
......@@ -5687,15 +4850,6 @@ version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
[[package]]
name = "nibble_vec"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43"
dependencies = [
"smallvec",
]
[[package]]
name = "nid"
version = "3.0.0"
......@@ -5826,15 +4980,6 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "ntapi"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "nu-ansi-term"
version = "0.50.3"
......@@ -5998,16 +5143,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "objc"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1"
dependencies = [
"malloc_buf",
"objc_exception",
]
[[package]]
name = "objc2"
version = "0.6.3"
......@@ -6167,15 +5302,6 @@ dependencies = [
"objc2-foundation",
]
[[package]]
name = "objc_exception"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4"
dependencies = [
"cc",
]
[[package]]
name = "object"
version = "0.37.3"
......@@ -6400,15 +5526,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "ordered-float"
version = "5.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f4779c6901a562440c3786d08192c6fbda7c1c2060edd10006b05ee35d10f2d"
dependencies = [
"num-traits",
]
[[package]]
name = "ordered-multimap"
version = "0.7.3"
......@@ -6435,16 +5552,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "packedvec"
version = "1.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a69e0a534dd2e6aefce319af62a0aa0066a76bdfcec0201dfe02df226bc9ec70"
dependencies = [
"num-traits",
"serde",
]
[[package]]
name = "parking"
version = "2.2.1"
......@@ -6599,7 +5706,6 @@ version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
dependencies = [
"phf_macros",
"phf_shared",
]
......@@ -6623,19 +5729,6 @@ dependencies = [
"rand 0.8.5",
]
[[package]]
name = "phf_macros"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro2",
"quote",
"syn 2.0.111",
]
[[package]]
name = "phf_shared"
version = "0.11.3"
......@@ -6773,12 +5866,6 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "precomputed-hash"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "prettyplease"
version = "0.2.37"
......@@ -6789,15 +5876,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "primal-check"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08"
dependencies = [
"num-integer",
]
[[package]]
name = "proc-macro-crate"
version = "3.4.0"
......@@ -7199,16 +6277,6 @@ version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "radix_trie"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd"
dependencies = [
"endian-type",
"nibble_vec",
]
[[package]]
name = "rand"
version = "0.8.5"
......@@ -7268,16 +6336,6 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rand_distr"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31"
dependencies = [
"num-traits",
"rand 0.8.5",
]
[[package]]
name = "rand_distr"
version = "0.5.1"
......@@ -7288,15 +6346,6 @@ dependencies = [
"rand 0.9.2",
]
[[package]]
name = "rand_isaac"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3382fc9f0aad4f2e2a56b53d9133c8c810b4dbf21e7e370e24346161a5b2c7bd"
dependencies = [
"rand_core 0.9.3",
]
[[package]]
name = "rand_xorshift"
version = "0.4.0"
......@@ -7411,27 +6460,12 @@ dependencies = [
"crossbeam-utils",
]
[[package]]
name = "realfft"
version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677"
dependencies = [
"rustfft",
]
[[package]]
name = "reborrow"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"
[[package]]
name = "recvmsg"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175"
[[package]]
name = "redox_syscall"
version = "0.5.18"
......@@ -7762,18 +6796,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "rubato"
version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5258099699851cfd0082aeb645feb9c084d9a5e1f1b8d5372086b989fc5e56a1"
dependencies = [
"num-complex",
"num-integer",
"num-traits",
"realfft",
]
[[package]]
name = "rust-embed"
version = "8.9.0"
......@@ -7819,34 +6841,14 @@ dependencies = [
]
[[package]]
name = "rust-mcp-schema"
version = "0.1.11"
name = "rustc-demangle"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69c8c97bf79c576f8dc582be9f6c9825ed91bd921aac65bd7990992257727e39"
dependencies = [
"serde",
"serde_json",
]
checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
[[package]]
name = "rust-stemmers"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
dependencies = [
"serde",
"serde_derive",
]
[[package]]
name = "rustc-demangle"
version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
[[package]]
name = "rustc-hash"
version = "1.1.0"
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
......@@ -7865,20 +6867,6 @@ dependencies = [
"semver",
]
[[package]]
name = "rustfft"
version = "6.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89"
dependencies = [
"num-complex",
"num-integer",
"num-traits",
"primal-check",
"strength_reduce",
"transpose",
]
[[package]]
name = "rustix"
version = "1.1.2"
......@@ -8055,15 +7043,6 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
[[package]]
name = "safe_arch"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"
dependencies = [
"bytemuck",
]
[[package]]
name = "safetensors"
version = "0.4.5"
......@@ -8074,16 +7053,6 @@ dependencies = [
"serde_json",
]
[[package]]
name = "safetensors"
version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "172dd94c5a87b5c79f945c863da53b2ebc7ccef4eca24ac63cca66a41aab2178"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "same-file"
version = "1.0.6"
......@@ -8111,18 +7080,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "schemars"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615"
dependencies = [
"dyn-clone",
"schemars_derive 0.8.22",
"serde",
"serde_json",
]
[[package]]
name = "schemars"
version = "0.9.0"
......@@ -8143,23 +7100,11 @@ checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289"
dependencies = [
"dyn-clone",
"ref-cast",
"schemars_derive 1.1.0",
"schemars_derive",
"serde",
"serde_json",
]
[[package]]
name = "schemars_derive"
version = "0.8.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
"syn 2.0.111",
]
[[package]]
name = "schemars_derive"
version = "1.1.0"
......@@ -8178,21 +7123,6 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "scraper"
version = "0.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "527e65d9d888567588db4c12da1087598d0f6f8b346cc2c5abc91f05fc2dffe2"
dependencies = [
"cssparser",
"ego-tree",
"getopts",
"html5ever 0.29.1",
"precomputed-hash",
"selectors",
"tendril",
]
[[package]]
name = "sdd"
version = "3.0.10"
......@@ -8245,25 +7175,6 @@ dependencies = [
"libc",
]
[[package]]
name = "selectors"
version = "0.26.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8"
dependencies = [
"bitflags 2.10.0",
"cssparser",
"derive_more 0.99.20",
"fxhash",
"log",
"new_debug_unreachable",
"phf",
"phf_codegen",
"precomputed-hash",
"servo_arc",
"smallvec",
]
[[package]]
name = "self_cell"
version = "1.2.1"
......@@ -8292,15 +7203,6 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde-big-array"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f"
dependencies = [
"serde",
]
[[package]]
name = "serde-untagged"
version = "0.1.9"
......@@ -8319,7 +7221,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c"
dependencies = [
"ordered-float 2.10.1",
"ordered-float",
"serde",
]
......@@ -8398,15 +7300,6 @@ dependencies = [
"serde_core",
]
[[package]]
name = "serde_plain"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50"
dependencies = [
"serde",
]
[[package]]
name = "serde_repr"
version = "0.1.20"
......@@ -8517,15 +7410,6 @@ dependencies = [
"syn 2.0.111",
]
[[package]]
name = "servo_arc"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "170fb83ab34de17dc69aa7c67482b22218ddb85da56546f9bd6b929e32a05930"
dependencies = [
"stable_deref_trait",
]
[[package]]
name = "sha1"
version = "0.10.6"
......@@ -8569,27 +7453,6 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "signal-hook"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2"
dependencies = [
"libc",
"signal-hook-registry",
]
[[package]]
name = "signal-hook-mio"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc"
dependencies = [
"libc",
"mio 0.8.11",
"signal-hook",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.7"
......@@ -8621,19 +7484,6 @@ dependencies = [
"rand_core 0.6.4",
]
[[package]]
name = "simba"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95"
dependencies = [
"approx",
"num-complex",
"num-traits",
"paste",
"wide",
]
[[package]]
name = "simd-adler32"
version = "0.3.7"
......@@ -8704,18 +7554,6 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "sparsevec"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68b4a8ce3045f0fe173fb5ae3c6b7dcfbec02bfa650bb8618b2301f52af0134d"
dependencies = [
"num-traits",
"packedvec",
"serde",
"vob",
]
[[package]]
name = "spin"
version = "0.9.8"
......@@ -8759,18 +7597,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "statrs"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e"
dependencies = [
"approx",
"nalgebra",
"num-traits",
"rand 0.8.5",
]
[[package]]
name = "stdio-override"
version = "0.2.0"
......@@ -8781,52 +7607,6 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "stop-words"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "645a3d441ccf4bf47f2e4b7681461986681a6eeea9937d4c3bc9febd61d17c71"
dependencies = [
"serde_json",
]
[[package]]
name = "strength_reduce"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
[[package]]
name = "string_cache"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f"
dependencies = [
"new_debug_unreachable",
"parking_lot",
"phf_shared",
"precomputed-hash",
"serde",
]
[[package]]
name = "string_cache_codegen"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0"
dependencies = [
"phf_generator",
"phf_shared",
"proc-macro2",
"quote",
]
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "strsim"
version = "0.11.1"
......@@ -8860,141 +7640,6 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symphonia"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5773a4c030a19d9bfaa090f49746ff35c75dfddfa700df7a5939d5e076a57039"
dependencies = [
"lazy_static",
"symphonia-bundle-flac",
"symphonia-bundle-mp3",
"symphonia-codec-pcm",
"symphonia-codec-vorbis",
"symphonia-core",
"symphonia-format-isomp4",
"symphonia-format-ogg",
"symphonia-format-riff",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-flac"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c91565e180aea25d9b80a910c546802526ffd0072d0b8974e3ebe59b686c9976"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-bundle-mp3"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4872dd6bb56bf5eac799e3e957aa1981086c3e613b27e0ac23b176054f7c57ed"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-codec-pcm"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e89d716c01541ad3ebe7c91ce4c8d38a7cf266a3f7b2f090b108fb0cb031d95"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-vorbis"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f025837c309cd69ffef572750b4a2257b59552c5399a5e49707cc5b1b85d1c73"
dependencies = [
"log",
"symphonia-core",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-core"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea00cc4f79b7f6bb7ff87eddc065a1066f3a43fe1875979056672c9ef948c2af"
dependencies = [
"arrayvec",
"bitflags 1.3.2",
"bytemuck",
"lazy_static",
"log",
]
[[package]]
name = "symphonia-format-isomp4"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "243739585d11f81daf8dac8d9f3d18cc7898f6c09a259675fc364b382c30e0a5"
dependencies = [
"encoding_rs",
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-ogg"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b4955c67c1ed3aa8ae8428d04ca8397fbef6a19b2b051e73b5da8b1435639cb"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
"symphonia-utils-xiph",
]
[[package]]
name = "symphonia-format-riff"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2d7c3df0e7d94efb68401d81906eae73c02b40d5ec1a141962c592d0f11a96f"
dependencies = [
"extended",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-metadata"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36306ff42b9ffe6e5afc99d49e121e0bd62fe79b9db7b9681d48e29fa19e6b16"
dependencies = [
"encoding_rs",
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-utils-xiph"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee27c85ab799a338446b68eec77abf42e1a6f1bb490656e121c6e27bfbab9f16"
dependencies = [
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "syn"
version = "1.0.109"
......@@ -9071,21 +7716,6 @@ dependencies = [
"walkdir",
]
[[package]]
name = "sysinfo"
version = "0.30.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3"
dependencies = [
"cfg-if 1.0.4",
"core-foundation-sys",
"libc",
"ntapi",
"once_cell",
"rayon",
"windows",
]
[[package]]
name = "system-configuration"
version = "0.5.1"
......@@ -9170,27 +7800,6 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "tendril"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
dependencies = [
"futf",
"mac",
"utf-8",
]
[[package]]
name = "terminal_size"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60b8cb979cb11c32ce1603f8137b22262a9d131aaa5c37b5678025f22b8becd0"
dependencies = [
"rustix",
"windows-sys 0.60.2",
]
[[package]]
name = "textwrap"
version = "0.11.0"
......@@ -9461,18 +8070,6 @@ dependencies = [
"tokio-stream",
]
[[package]]
name = "tokio-tungstenite"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.24.0",
]
[[package]]
name = "tokio-tungstenite"
version = "0.26.2"
......@@ -9482,7 +8079,7 @@ dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.26.2",
"tungstenite",
]
[[package]]
......@@ -9522,18 +8119,6 @@ dependencies = [
"webpki-roots 0.26.11",
]
[[package]]
name = "toktrie"
version = "1.0.0"
source = "git+https://github.com/guidance-ai/llguidance.git?rev=c432092#c432092d37b8ccd1afeeff3e7f9c9a29aae0a87e"
dependencies = [
"anyhow",
"bytemuck",
"bytemuck_derive",
"serde",
"serde_json",
]
[[package]]
name = "toktrie"
version = "1.4.0"
......@@ -9547,19 +8132,6 @@ dependencies = [
"serde_json",
]
[[package]]
name = "toktrie_hf_tokenizers"
version = "1.0.0"
source = "git+https://github.com/guidance-ai/llguidance.git?rev=c432092#c432092d37b8ccd1afeeff3e7f9c9a29aae0a87e"
dependencies = [
"anyhow",
"log",
"serde",
"serde_json",
"tokenizers",
"toktrie 1.0.0",
]
[[package]]
name = "toktrie_hf_tokenizers"
version = "1.4.0"
......@@ -9571,7 +8143,7 @@ dependencies = [
"serde",
"serde_json",
"tokenizers",
"toktrie 1.4.0",
"toktrie",
]
[[package]]
......@@ -9872,17 +8444,6 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tqdm"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa2d2932240205a99b65f15d9861992c95fbb8c9fb280b3a1f17a92db6dc611f"
dependencies = [
"anyhow",
"crossterm",
"once_cell",
]
[[package]]
name = "tracing"
version = "0.1.43"
......@@ -9978,16 +8539,6 @@ dependencies = [
"tracing-serde",
]
[[package]]
name = "transpose"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e"
dependencies = [
"num-integer",
"strength_reduce",
]
[[package]]
name = "try-lock"
version = "0.2.5"
......@@ -10004,24 +8555,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "tungstenite"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a"
dependencies = [
"byteorder",
"bytes",
"data-encoding",
"http 1.4.0",
"httparse",
"log",
"rand 0.8.5",
"sha1",
"thiserror 1.0.69",
"utf-8",
]
[[package]]
name = "tungstenite"
version = "0.26.2"
......@@ -10067,61 +8600,13 @@ dependencies = [
"num-traits",
"num_cpus",
"rayon",
"safetensors 0.4.5",
"safetensors",
"serde",
"thiserror 1.0.69",
"tracing",
"yoke 0.7.5",
]
[[package]]
name = "ug"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76b761acf8af3494640d826a8609e2265e19778fb43306c7f15379c78c9b05b0"
dependencies = [
"gemm 0.18.2",
"half 2.7.1",
"libloading",
"memmap2",
"num",
"num-traits",
"num_cpus",
"rayon",
"safetensors 0.4.5",
"serde",
"thiserror 1.0.69",
"tracing",
"yoke 0.7.5",
]
[[package]]
name = "ug-cuda"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f0a1fa748f26166778c33b8498255ebb7c6bffb472bcc0a72839e07ebb1d9b5"
dependencies = [
"cudarc",
"half 2.7.1",
"serde",
"thiserror 1.0.69",
"ug 0.5.0",
]
[[package]]
name = "ug-metal"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7adf545a99a086d362efc739e7cf4317c18cbeda22706000fd434d70ea3d95"
dependencies = [
"half 2.7.1",
"metal 0.29.0",
"objc",
"serde",
"thiserror 1.0.69",
"ug 0.5.0",
]
[[package]]
name = "unarray"
version = "0.1.4"
......@@ -10329,12 +8814,6 @@ dependencies = [
"serde",
]
[[package]]
name = "urlencoding"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf-8"
version = "0.7.6"
......@@ -10376,7 +8855,7 @@ dependencies = [
"regex",
"syn 2.0.111",
"url",
"uuid 1.18.1",
"uuid",
]
[[package]]
......@@ -10397,15 +8876,6 @@ dependencies = [
"zip 3.0.0",
]
[[package]]
name = "uuid"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
dependencies = [
"getrandom 0.2.16",
]
[[package]]
name = "uuid"
version = "1.18.1"
......@@ -10465,63 +8935,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
[[package]]
name = "variantly"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72a332341ba79a179d9e9b33c0d72fbf3dc2c80e1be79416401a08d2b820ef56"
dependencies = [
"Inflector",
"darling 0.11.0",
"proc-macro2",
"quote",
"syn 1.0.109",
"uuid 0.8.2",
]
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vergen"
version = "9.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b2bf58be11fc9414104c6d3a2e464163db5ef74b12296bda593cac37b6e4777"
dependencies = [
"anyhow",
"derive_builder",
"rustversion",
"vergen-lib",
]
[[package]]
name = "vergen-gitcl"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9dfc1de6eb2e08a4ddf152f1b179529638bedc0ea95e6d667c014506377aefe"
dependencies = [
"anyhow",
"derive_builder",
"rustversion",
"time",
"vergen",
"vergen-lib",
]
[[package]]
name = "vergen-lib"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b07e6010c0f3e59fcb164e0163834597da68d1f864e2b8ca49f74de01e9c166"
dependencies = [
"anyhow",
"derive_builder",
"rustversion",
]
[[package]]
name = "version-compare"
version = "0.2.1"
......@@ -10551,16 +8970,6 @@ version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "vob"
version = "3.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc936b5a7202a703aeaf7ce05e7931db2e0c8126813f97db3e9e06d867b0bb38"
dependencies = [
"num-traits",
"serde",
]
[[package]]
name = "wait-timeout"
version = "0.2.1"
......@@ -10695,18 +9104,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web_atoms"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57ffde1dc01240bdf9992e3205668b235e59421fd085e8a317ed98da0178d414"
dependencies = [
"phf",
"phf_codegen",
"string_cache",
"string_cache_codegen",
]
[[package]]
name = "webpki-roots"
version = "0.26.11"
......@@ -10731,22 +9128,6 @@ version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
[[package]]
name = "wide"
version = "0.7.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"
dependencies = [
"bytemuck",
"safe_arch",
]
[[package]]
name = "widestring"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471"
[[package]]
name = "winapi"
version = "0.2.8"
......@@ -10790,25 +9171,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be"
dependencies = [
"windows-core 0.52.0",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-core"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-core"
version = "0.62.2"
......@@ -11298,7 +9660,7 @@ dependencies = [
"thiserror 1.0.69",
"tokio",
"tokio-util",
"uuid 1.18.1",
"uuid",
]
[[package]]
......
Cargo.toml
View file @ bbe82f18
......@@ -3,7 +3,6 @@
[workspace]
members = [
"launch/dynamo-run",
"lib/llm",
"lib/runtime",
"lib/config",
......@@ -15,13 +14,10 @@ members = [
"lib/parsers",
"lib/bindings/c",
"lib/bindings/python/codegen",
"lib/engines/*",
"lib/config",
]
# Exclude certain packages that are slow to build and we don't ship as flagship
# features from default build, but keep them in workspace for convenience.
# - launch/dynamo-run
# - lib/engines/*
default-members = [
"lib/llm",
"lib/runtime",
......
components/src/dynamo/sglang/protocol.py
View file @ bbe82f18
......@@ -14,7 +14,6 @@ TokenIdType = int
# ============================================================================
# Standard LLM Protocol Types
# ============================================================================
# TODO: move these to common for all LLMs once we adopt dynamo-run
# derived from lib/llm/src/protocols/common/preprocessor.rs
class StopConditions(BaseModel):
max_tokens: Optional[int] = None
......
container/templates/wheel_builder.Dockerfile
View file @ bbe82f18
......@@ -326,7 +326,6 @@ RUN --mount=type=secret,id=aws-key-id,env=AWS_ACCESS_KEY_ID \
# Copy source code (order matters for layer caching)
COPY pyproject.toml README.md LICENSE Cargo.toml Cargo.lock rust-toolchain.toml hatch_build.py /opt/dynamo/
COPY launch/ /opt/dynamo/launch/
COPY lib/ /opt/dynamo/lib/
COPY components/ /opt/dynamo/components/
......
docs/pages/backends/trtllm/multinode/multinode-examples.md
View file @ bbe82f18
......@@ -200,7 +200,6 @@ deployment across 8 nodes:
2. The OpenAI frontend will listen for and dynamically discover workers as
they register themselves with Dynamo's distributed runtime:
```
0: 2025-06-13T02:36:48.160Z INFO dynamo_run::input::http: Watching for remote model at models
0: 2025-06-13T02:36:48.161Z INFO dynamo_llm::http::service::service_v2: Starting HTTP service on: 0.0.0.0:8000 address="0.0.0.0:8000"
```
3. The TRTLLM worker will consist of N (N=16 for TP16) MPI ranks, 1 rank on each
......
docs/pages/kubernetes/deployment/create-deployment.md
View file @ bbe82f18
......@@ -147,7 +147,6 @@ Consult the corresponding sh file. Each of the python commands to launch a compo
The front end is launched with "python3 -m dynamo.frontend [--http-port 8000] [--router-mode kv]"
Each worker will launch `python -m dynamo.YOUR_INFERENCE_BACKEND --model YOUR_MODEL --your-flags `command.
If you are a Dynamo contributor the [dynamo run guide](../../reference/cli.md) for details on how to run this command.
## Step 3: Key Customization Points
......
docs/pages/reference/cli.md deleted 100644 → 0
View file @ 2c747d64
---
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
---
`dynamo-run` is a Rust binary that lets you easily run a model, explore the Dynamo components, and demonstrates the Rust API. It supports the `mistral.rs` engines, as well as testing engines `echo` and `mocker`.
It is primarily for development and rapid prototyping. For production use we recommend the Python wrapped components, see the main project README.
## Basics
Usage: See `dynamo-run --help`
Example: `dynamo-run Qwen/Qwen3-0.6B`
Set the environment variable `DYN_LOG` to adjust the logging level; for example, `export DYN_LOG=debug`. It has the same syntax as `RUST_LOG`.
To adjust verbosity, use `-v` to enable debug logging or `-vv` to enable full trace logging. For example:
```bash
dynamo-run in=http out=mistralrs <model> -v # enables debug logging
```
### Use model from Hugging Face
To automatically download Qwen3 4B from Hugging Face (16 GiB download) and to start it in interactive text mode:
```
dynamo-run Qwen/Qwen3-4B
```
The general format for HF download follows this pattern:
```
dynamo-run out=<engine> <HUGGING_FACE_ORGANIZATION/MODEL_NAME>
```
For gated models (such as meta-llama/Llama-3.2-3B-Instruct), you must set an `HF_TOKEN` environment variable.
The parameter can be the ID of a HuggingFace repository (which will be downloaded) or a folder containing safetensors, config.json, or similar (perhaps a locally checked out HuggingFace repository).
### Run a model from local file
To run a model from local file:
- Download the model from Hugging Face
- Run the model from local file
See the following sections for details.
#### Download model from Hugging Face
This model available from Hugging Face should be high quality and fast on almost any machine: https://huggingface.co/Qwen/Qwen3-0.6B
To run the model:
*Text interface*
```
dynamo-run Qwen/Qwen3-0.6B
```
You can also pipe a prompt into `dynamo-run`:
```
echo 'What is the capital of Tuvalu?' | dynamo-run Qwen/Qwen3-0.6B --context-length 4096
```
*HTTP interface*
```
dynamo-run in=http out=mistralrs Qwen/Qwen3-0.6B
```
You can also list models or send a request:
*List the models*
```
curl localhost:8080/v1/models
```
*Send a request*
```
curl -d '{"model": "Qwen/Qwen3-0.6B", "max_completion_tokens": 2049, "messages":[{"role":"user", "content": "What is the capital of South Africa?" }]}' -H 'Content-Type: application/json' http://localhost:8080/v1/chat/completions
```
## Distributed System
You can run the ingress side (HTTP server and pre-processing) on one machine, for example a CPU node, and the worker on a different machine (a GPU node).
You will need [etcd](https://etcd.io/) and [nats](https://nats.io) with jetstream installed and accessible from both nodes. For development I run NATS like this: `nats-server -js --trace --store_dir $(mktemp -d)`.
**Node 1:** OpenAI compliant HTTP server, optional pre-processing, worker discovery:
```
dynamo-run in=http out=auto
```
**Node 2:** Engine. Receives and returns requests over the network:
```
dynamo-run in=dyn://llama3B.backend.generate out=mistralrs ~/llms/Llama-3.2-3B-Instruct
```
This uses etcd to auto-discover the model and NATS to talk to it. You can
run multiple instances on the same endpoint; it picks one based on the
`--router-mode` (round-robin by default if left unspecified).
Run `dynamo-run --help` for more options.
### Network names
The `in=dyn://` URLs have the format `dyn://namespace.component.endpoint`. For quickstart just use any string `dyn://test`, `dynamo-run` will default any missing parts for you. The pieces matter for a larger system.
* *Namespace*: A pipeline. Usually a model. e.g "llama_8b". Just a name.
* *Component*: A load balanced service needed to run that pipeline. "backend", "prefill", "decode", "preprocessor", "draft", etc. This typically has some configuration (which model to use, for example).
* *Endpoint*: Like a URL. "generate", "load_metrics".
* *Instance*: A process. Unique. Dynamo assigns each one a unique instance_id. The thing that is running is always an instance. Namespace/component/endpoint can refer to multiple instances.
If you run two models, that is two pipelines. An exception would be if doing speculative decoding. The draft model is part of the pipeline of a bigger model.
If you run two instances of the same model ("data parallel") they are the same namespace+component+endpoint but different instances. The router will spread traffic over all the instances of a namespace+component+endpoint. If you have four prefill workers in a pipeline, they all have the same namespace+component+endpoint and are automatically assigned unique instance_ids.
Example 1: Data parallel load balanced, one model one pipeline two instances.
```
Node 1: dynamo-run in=dyn://qwen3-32b.backend.generate /data/Qwen3-32B
Node 2: dynamo-run in=dyn://qwen3-32b.backend.generate /data/Qwen3-32B
```
Example 2: Two models, two pipelines.
```
Node 1: dynamo-run in=dyn://qwen3-32b.backend.generate /data/Qwen3-32B
Node 2: dynamo-run in=dyn://llama3-1-8b.backend.generate /data/Llama-3.1-8B-Instruct/
```
Example 3: Different endpoints.
The KV metrics publisher in VLLM adds a `load_metrics` endpoint to the current component. If the `llama3-1-8b.backend` component above is using patched vllm it will also expose `llama3-1-8b.backend.load_metrics`.
Example 4: Multiple component in a pipeline.
In the P/D disaggregated setup you would have `deepseek-distill-llama8b.prefill.generate` (possibly multiple instances of this) and `deepseek-distill-llama8b.decode.generate`.
For output it is always only `out=auto`. This tells Dynamo to auto-discover the instances, group them by model, and load balance appropriately (depending on `--router-mode` flag).
### KV-aware routing
```
dynamo-run in=http out=auto --router-mode kv
```
The only difference from the distributed system above is `--router-mode kv`. vllm announces when a KV block is created or removed. The Dynamo router finds the worker with the best match for those KV blocks and directs the traffic to that node.
For performance testing, compare a typical workload with `--router-mode random|round-robin` to see if it can benefit from KV-aware routing.
The KV-aware routing arguments:
- `--kv-overlap-score-weight`: Sets the amount of weighting on overlaps with prefix caches, which directly contributes to the prefill cost. A large weight is expected to yield a better TTFT (at the expense of worse ITL). When set to 0, prefix caches are not considered at all (falling back to pure load balancing behavior on the active blocks).
- `--router-temperature`: Sets the temperature when randomly selecting workers to route to via softmax sampling on the router cost logits. Setting it to 0 recovers the deterministic behavior where the min logit is picked.
- `--use-kv-events`: Sets whether to listen to KV events for maintaining the global view of cached blocks. If true, the router uses KV events to track block creation and deletion from workers. If false, the router predicts cache state based on routing decisions with TTL-based expiration (default 120s) and pruning. Set false if your backend engine does not emit KV events.
### Request Migration
In a [Distributed System](#distributed-system), you can enable [request migration](../fault-tolerance/request-migration.md) to handle worker failures gracefully. Use the `--migration-limit` flag to specify how many times a request can be migrated to another worker:
```bash
dynamo-run in=dyn://... out=<engine> ... --migration-limit=3
```
This allows a request to be migrated up to 3 times before failing. See the [Request Migration Architecture](../fault-tolerance/request-migration.md) documentation for details on how this works.
### Request Cancellation
When using the HTTP interface (`in=http`), if the HTTP request connection is dropped by the client, Dynamo automatically cancels the downstream request to the worker. This ensures that computational resources are not wasted on generating responses that are no longer needed.
For detailed information about how request cancellation works across the system, see the [Request Cancellation Architecture](../fault-tolerance/request-cancellation.md) documentation.
## Development
`dynamo-run` is also an example of what can be built in Rust with the `dynamo-llm` and `dynamo-runtime` crates. The following guide shows how to build from source with all the features.
### Step 1: Install libraries
**Ubuntu:**
```
sudo apt install -y build-essential libhwloc-dev libudev-dev pkg-config libssl-dev libclang-dev protobuf-compiler python3-dev cmake
```
**macOS:**
- [Homebrew](https://brew.sh/)
```
# if brew is not installed on your system, install it
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
```
- [Xcode](https://developer.apple.com/xcode/)
```
brew install cmake protobuf
## Check that Metal is accessible
xcrun -sdk macosx metal
```
If Metal is accessible, you should see an error like `metal: error: no input files`, which confirms it is installed correctly.
### Step 2: Install Rust
```
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
source $HOME/.cargo/env
```
### Step 3: Build
- Linux with GPU and CUDA (tested on Ubuntu):
```
cargo build --features cuda
```
- macOS with Metal:
```
cargo build --features metal
```
- CPU only:
```
cargo build
```
Optionally you can run `cargo build` from any location with arguments:
```
--target-dir /path/to/target_directory # specify target_directory with write privileges
--manifest-path /path/to/project/Cargo.toml # if cargo build is run outside of `launch/` directory
```
The binary is called `dynamo-run` in `target/debug`
```
cd target/debug
```
Build with `--release` for a smaller binary and better performance, but longer build times. The binary will be in `target/release`.
## Engines
The input defaults to `in=text`. The output defaults to `out=mistralrs` engine, unless it is disabled with `--no-default-features` in which case an engine that echo's back your input is used.
### mistralrs
[mistral.rs](https://github.com/EricLBuehler/mistral.rs) is a pure Rust engine that is fast to run and fast to load, and runs well on CPU as well as GPU. For those reasons it is the default engine.
```
dynamo-run Qwen/Qwen3-4B
```
is equivalent to
```
dynamo-run in=text out=mistralrs Qwen/Qwen3-4B
```
If you have multiple GPUs, `mistral.rs` does automatic tensor parallelism. You do not need to pass any extra flags to dynamo-run to enable it.
### Mocker engine
The mocker engine is a mock vLLM implementation designed for testing and development purposes. It simulates realistic token generation timing without requiring actual model inference, making it useful for:
- Testing distributed system components without GPU resources
- Benchmarking infrastructure and networking overhead
- Developing and debugging Dynamo components
- Load testing and performance analysis
**Basic usage:**
The `--model-path` is required but can point to any valid model path - the mocker doesn't actually load the model weights (but the pre-processor needs the tokenizer). The arguments `block_size`, `num_gpu_blocks`, `max_num_seqs`, `max_num_batched_tokens`, `enable_prefix_caching`, and `enable_chunked_prefill` are common arguments shared with the real VLLM engine.
And below are arguments that are mocker-specific:
- `speedup_ratio`: Speed multiplier for token generation (default: 1.0). Higher values make the simulation engines run faster.
- `dp_size`: Number of data parallel workers to simulate (default: 1)
- `watermark`: KV cache watermark threshold as a fraction (default: 0.01). This argument also exists for the real VLLM engine but cannot be passed as an engine arg.
```bash
echo '{"speedup_ratio": 10.0}' > mocker_args.json
dynamo-run in=dyn://dynamo.mocker.generate out=mocker --model-path TinyLlama/TinyLlama-1.1B-Chat-v1.0 --extra-engine-args mocker_args.json
dynamo-run in=http out=auto --router-mode kv
```
### echo
The `echo` engine echoes the prompt back as the response.
```
dynamo-run in=http out=echo --model-name my_model
```
The echo engine uses a configurable delay between tokens to simulate generation speed. You can adjust this using the `DYN_TOKEN_ECHO_DELAY_MS` environment variable:
```
# Set token echo delay to 1ms (1000 tokens per second)
DYN_TOKEN_ECHO_DELAY_MS=1 dynamo-run in=http out=echo
```
The default delay is 10ms, which produces approximately 100 tokens per second.
### Other engines, multi-node, production
`vllm`, `sglang` and `trtllm` production grade engines are available in `examples/backends`. They run as Python components, using the Rust bindings. See the main README.
`dynamo-run` is an exploration, development and prototyping tool, as well as an example of using the Rust API. Multi-node and production setups should be using the main engine components.
## Batch mode
`dynamo-run` can take a jsonl file full of prompts and evaluate them all:
```
dynamo-run in=batch:prompts.jsonl out=mistralrs <model>
```
The input file should look like this:
```
{"text": "What is the capital of France?"}
{"text": "What is the capital of Spain?"}
```
Each one is passed as a prompt to the model. The output is written back to the same folder in `output.jsonl`. At the end of the run some statistics are printed.
The output looks like this:
```
{"text":"What is the capital of France?","response":"The capital of France is Paris.","tokens_in":7,"tokens_out":7,"elapsed_ms":1566}
{"text":"What is the capital of Spain?","response":".The capital of Spain is Madrid.","tokens_in":7,"tokens_out":7,"elapsed_ms":855}
```
## Writing your own engine in Python
The [dynamo](https://pypi.org/project/ai-dynamo/) Python library allows you to build your own engine and attach it to Dynamo. All of the main backend components in `examples/backends/` work like this.
The Python file must do three things:
1. Decorate a function to get the runtime
2. Register on the network
3. Attach a request handler
```
from dynamo.llm import ModelInput, ModelType, register_llm
from dynamo.runtime import DistributedRuntime, dynamo_worker
# 1. Decorate a function to get the runtime
#
@dynamo_worker()
async def worker(runtime: DistributedRuntime):
# 2. Register ourselves on the network
#
component = runtime.namespace("namespace").component("component")
model_path = "Qwen/Qwen3-0.6B" # or "/data/models/Qwen3-0.6B"
model_input = ModelInput.Tokens # or ModelInput.Text if engine handles pre-processing
model_type = ModelType.Chat # or ModelType.Chat | ModelType.Completions if model can be deployed on chat and completions endpoints
endpoint = component.endpoint("endpoint")
# Optional last param to register_llm is model_name. If not present derives it from model_path
await register_llm(model_input, model_type, endpoint, model_path)
# Initialize your engine here
# engine = ...
# 3. Attach request handler
#
await endpoint.serve_endpoint(RequestHandler(engine).generate)
class RequestHandler:
def __init__(self, engine):
...
async def generate(self, request):
# Call the engine
# yield result dict
...
if __name__ == "__main__":
uvloop.install()
asyncio.run(worker())
```
The `model_path` can be:
- A HuggingFace repo ID, optionally prefixed with `hf://`. It is downloaded and cached locally.
- The path to a checkout of a HuggingFace repo - any folder containing safetensor files as well as `config.json`, `tokenizer.json` and `tokenizer_config.json`.
The `model_input` can be:
- ModelInput.Tokens. Your engine expects pre-processed input (token IDs). Dynamo handles tokenization and pre-processing.
- ModelInput.Text. Your engine expects raw text input and handles its own tokenization and pre-processing.
The `model_type` can be:
- ModelType.Chat. Your `generate` method receives a `request` and must return a response dict of type [OpenAI Chat Completion](https://platform.openai.com/docs/api-reference/chat).
- ModelType.Completions. Your `generate` method receives a `request` and must return a response dict of the older [Completions](https://platform.openai.com/docs/api-reference/completions).
`register_llm` can also take the following kwargs:
- `model_name`: The name to call the model. Your incoming HTTP requests model name must match this. Defaults to the hugging face repo name, or the folder name.
- `context_length`: Max model length in tokens. Defaults to the model's set max. Only set this if you need to reduce KV cache allocation to fit into VRAM.
- `kv_cache_block_size`: Size of a KV block for the engine, in tokens. Defaults to 16.
- `user_data`: Optional dictionary containing custom metadata for worker behavior (e.g., LoRA configuration). Defaults to None.
Here are some example engines:
- Backend:
* [vllm](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/examples/hello_world/server_vllm.py)
* [sglang](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/examples/hello_world/server_sglang.py)
- Chat:
* [sglang](https://github.com/ai-dynamo/dynamo/blob/main/lib/bindings/python/examples/hello_world/server_sglang_tok.py)
More fully-featured Python engines are in `examples/backends`.
## Debugging
`dynamo-run` and `dynamo-runtime` support [tokio-console](https://github.com/tokio-rs/console). Build with the feature to enable:
```
cargo build --features cuda,tokio-console -p dynamo-run
```
The listener uses the default tokio console port, and all interfaces (0.0.0.0).
docs/versions/next.yml
View file @ bbe82f18
......@@ -253,8 +253,6 @@ navigation:
- page: Model Caching with Fluid
path: ../pages/kubernetes/model-caching-with-fluid.md
# -- Reference --
- page: CLI Reference
path: ../pages/reference/cli.md
- page: Glossary
path: ../pages/reference/glossary.md
- page: Tuning Disaggregated Performance
......
dynamo.code-workspace
View file @ bbe82f18
......@@ -10,7 +10,6 @@
"settings": {
"rust-analyzer.linkedProjects": [
"Cargo.toml",
"launch/dynamo-run/Cargo.toml",
"lib/bindings/python/Cargo.toml"
],
"rust-analyzer.procMacro.enable": true,
......
launch/dynamo-run/Cargo.toml deleted 100644 → 0
View file @ 2c747d64
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
[package]
name = "dynamo-run"
version.workspace = true
edition.workspace = true
authors.workspace = true
homepage.workspace = true
license.workspace = true
repository.workspace = true
readme.workspace = true
description = "Dynamo Run CLI"
[features]
# Build with `--no-default-features` to disable these defaults
default = ["mistralrs"]
mistralrs = ["dep:dynamo-engine-mistralrs"]
cuda = ["dynamo-engine-mistralrs/cuda"]
metal = ["dynamo-engine-mistralrs/metal"]
tokio-console = ["dynamo-runtime/tokio-console"]
[dependencies]
dynamo-llm = { workspace = true }
dynamo-runtime = { workspace = true }
dynamo-engine-mistralrs = { path = "../../lib/engines/mistralrs", optional = true }
anyhow = { workspace = true }
dynamo-async-openai = { workspace = true }
async-stream = { workspace = true }
async-trait = { workspace = true }
either = { workspace = true }
futures = { workspace = true }
libc = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
uuid = { workspace = true }
clap = { version = "4.5", features = ["derive", "env"] }
futures-util = { version = "0.3" }
regex = "1"
[build-dependencies]
vergen-gitcl = "1.0"
anyhow = { workspace = true }
launch/dynamo-run/README.md deleted 100644 → 0
View file @ 2c747d64
See `docs/guides/dynamo_run.md`
launch/dynamo-run/build.rs deleted 100644 → 0
View file @ 2c747d64
// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use std::env;
use std::process::Command;
use vergen_gitcl::{Emitter, GitclBuilder};
fn main() -> anyhow::Result<()> {
if has_cuda_toolkit() && !has_feature("cuda") && is_cuda_engine() {
println!("cargo:warning=CUDA not enabled, re-run with `--features cuda`");
}
if is_mac() && !has_feature("metal") {
println!("cargo:warning=Metal not enabled, re-run with `--features metal`");
}
let git_config = GitclBuilder::default()
.describe(true, false, None)
.build()?;
Emitter::default().add_instructions(&git_config)?.emit()?;
Ok(())
}
fn has_feature(s: &str) -> bool {
env::var(format!("CARGO_FEATURE_{}", s.to_uppercase())).is_ok()
}
fn has_cuda_toolkit() -> bool {
if let Ok(output) = Command::new("nvcc").arg("--version").output() {
output.status.success()
} else {
false
}
}
fn is_cuda_engine() -> bool {
has_feature("mistralrs")
}
#[cfg(target_os = "macos")]
fn is_mac() -> bool {
true
}
#[cfg(not(target_os = "macos"))]
fn is_mac() -> bool {
false
}
launch/dynamo-run/src/flags.rs deleted 100644 → 0
View file @ 2c747d64
// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use std::collections::HashMap;
use std::path::PathBuf;
use clap::ValueEnum;
use dynamo_llm::entrypoint::RouterConfig;
use dynamo_llm::kv_router::KvRouterConfig;
use dynamo_llm::mocker::protocols::MockEngineArgs;
use dynamo_runtime::pipeline::RouterMode as RuntimeRouterMode;
use crate::Output;
/// Required options depend on the in and out choices
#[derive(clap::Parser, Debug, Clone)]
#[command(version, about, long_about = None)]
pub struct Flags {
/// The model. The options depend on the engine.
///
/// The full list - only mistralrs supports all three currently:
/// - Full path of a checked out Hugging Face repository containing safetensor files
/// - Name of a Hugging Face repository, e.g 'google/flan-t5-small'. The model will be
/// downloaded and cached.
#[arg(index = 1)]
pub model_path_pos: Option<PathBuf>,
// `--model-path`. The one above is `dynamo-run <positional-model-path>`
#[arg(long = "model-path")]
pub model_path_flag: Option<PathBuf>,
/// HTTP port. `in=http` only
/// If tls_cert_path and tls_key_path are provided, this will be TLS/HTTPS.
#[arg(long, default_value = "8000")]
pub http_port: u16,
/// TLS certificate file
#[arg(long, requires = "tls_key_path")]
pub tls_cert_path: Option<PathBuf>,
/// TLS certificate key file
#[arg(long, requires = "tls_cert_path")]
pub tls_key_path: Option<PathBuf>,
/// The name of the model we are serving
#[arg(long)]
pub model_name: Option<String>,
/// Verbose output (-v for debug, -vv for trace)
#[arg(short = 'v', action = clap::ArgAction::Count, default_value_t = 0)]
pub verbosity: u8,
/// If using `out=dyn` with multiple instances, this says how to route the requests.
///
/// Mostly interesting for KV-aware routing.
/// Defaults to RouterMode::RoundRobin
#[arg(long, default_value = "round-robin")]
pub router_mode: RouterMode,
/// KV Router: Weight for overlap score in worker selection.
/// Higher values prioritize KV cache reuse. Default: 1.0
#[arg(long)]
pub kv_overlap_score_weight: Option<f64>,
/// KV Router: Temperature for worker sampling via softmax.
/// Higher values promote more randomness, and 0 fallbacks to deterministic.
/// Default: 0.0
#[arg(long)]
pub router_temperature: Option<f64>,
/// KV Router: Whether to use KV events to maintain the view of cached blocks
/// If false, the router predicts cache state based on routing decisions
/// with TTL-based expiration and pruning, rather than receiving events from workers.
/// Default: true
#[arg(long)]
pub use_kv_events: Option<bool>,
/// KV Router: Whether to enable replica synchronization across multiple router instances.
/// When true, routers will publish and subscribe to events to maintain consistent state.
/// Default: false
#[arg(long)]
pub router_replica_sync: Option<bool>,
/// KV Router: Whether to track active blocks in the router for memory management.
/// When false, the router will not maintain state about which blocks are active,
/// reducing memory overhead but potentially affecting scheduling decisions.
/// Default: true
#[arg(long)]
pub router_track_active_blocks: Option<bool>,
/// Max model context length. Reduce this if you don't have enough VRAM for the full model
/// context length (e.g. Llama 4).
/// Defaults to the model's max, which is usually model_max_length in tokenizer_config.json.
#[arg(long)]
pub context_length: Option<u32>,
/// KV cache block size (is this used? Maybe by Python vllm worker?)
#[arg(long)]
pub kv_cache_block_size: Option<u32>,
/// Mocker engine only.
/// Additional engine-specific arguments from a JSON file.
/// Contains a mapping of parameter names to values.
#[arg(long)]
pub extra_engine_args: Option<PathBuf>,
/// Path to a JSON file containing default request fields.
/// These fields will be merged with each request, but can be overridden by the request.
/// Example file contents:
/// {
/// "model": "Qwen2.5-3B-Instruct",
/// "temperature": 0.7,
/// "max_completion_tokens": 4096
/// }
#[arg(long)]
pub request_template: Option<PathBuf>,
/// How many times a request can be migrated to another worker if the HTTP server lost
/// connection to the current worker.
#[arg(long, value_parser = clap::value_parser!(u32).range(0..1024))]
pub migration_limit: Option<u32>,
/// Which key-value backend to use: etcd, mem, file.
/// Etcd uses the ETCD_* env vars (e.g. ETCD_ENDPOINTS) for connection details.
/// File uses root dir from env var DYN_FILE_KV or defaults to $TMPDIR/dynamo_store_kv.
#[arg(long, default_value = "etcd", value_parser = ["etcd", "file", "mem"])]
pub store_kv: String,
/// Determines how requests are distributed from routers to workers. 'tcp' is fastest [nats|http|tcp].
#[arg(long, default_value = "tcp", value_parser = ["nats", "http", "tcp"])]
pub request_plane: String,
/// Everything after a `--`. Not currently used.
#[arg(index = 2, last = true, hide = true, allow_hyphen_values = true)]
pub last: Vec<String>,
}
impl Flags {
/// For each Output variant, check if it would be able to run.
/// This takes validation out of the main engine creation path.
pub fn validate(&self, out_opt: &Output) -> anyhow::Result<()> {
match out_opt {
Output::Auto => {
if self.context_length.is_some() {
anyhow::bail!(
"'--context-length' flag should only be used on the worker node, not on the ingress"
);
}
if self.kv_cache_block_size.is_some() {
anyhow::bail!(
"'--kv-cache-block-size' flag should only be used on the worker node, not on the ingress"
);
}
}
Output::Echo => {}
#[cfg(feature = "mistralrs")]
Output::MistralRs => {}
Output::Mocker => {
// nothing to check here
}
}
match out_opt {
Output::Mocker => {}
_ => {
if self.extra_engine_args.is_some() {
anyhow::bail!("`--extra-engine-args` is only for the mocker engine");
}
}
}
Ok(())
}
pub fn router_config(&self) -> RouterConfig {
let mut cfg = KvRouterConfig::default();
if let Some(w) = self.kv_overlap_score_weight {
cfg.overlap_score_weight = w;
}
if let Some(t) = self.router_temperature {
cfg.router_temperature = t;
}
if let Some(v) = self.use_kv_events {
cfg.use_kv_events = v;
}
if let Some(v) = self.router_replica_sync {
cfg.router_replica_sync = v;
}
if let Some(v) = self.router_track_active_blocks {
cfg.router_track_active_blocks = v;
}
RouterConfig::new(self.router_mode.into(), cfg)
}
/// Load extra engine arguments from a JSON file
/// Returns a HashMap of parameter names to values
pub fn load_extra_engine_args(
&self,
) -> anyhow::Result<Option<HashMap<String, serde_json::Value>>> {
if let Some(path) = &self.extra_engine_args {
let file_content = std::fs::read_to_string(path)?;
let args: HashMap<String, serde_json::Value> = serde_json::from_str(&file_content)?;
Ok(Some(args))
} else {
Ok(None)
}
}
pub fn mocker_config(&self) -> MockEngineArgs {
let Some(path) = &self.extra_engine_args else {
tracing::warn!("Did not specify extra engine args. Using default mocker args.");
return MockEngineArgs::default();
};
MockEngineArgs::from_json_file(path)
.unwrap_or_else(|e| panic!("Failed to build mocker engine args from {path:?}: {e}"))
}
}
#[derive(Default, PartialEq, Eq, ValueEnum, Clone, Debug, Copy)]
pub enum RouterMode {
#[default]
#[value(name = "round-robin")]
RoundRobin,
Random,
#[value(name = "kv")]
KV,
}
impl From<RouterMode> for RuntimeRouterMode {
fn from(r: RouterMode) -> RuntimeRouterMode {
match r {
RouterMode::RoundRobin => RuntimeRouterMode::RoundRobin,
RouterMode::Random => RuntimeRouterMode::Random,
RouterMode::KV => RuntimeRouterMode::KV,
}
}
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment