Skip to content

GitLab

Unverified Commit 07207a1b authored by Graham King's avatar Graham King Committed by GitHub
Browse files

chore: Upgrade mistralrs (#3490) 


Signed-off-by: default avatarGraham King <grahamk@nvidia.com>
parent 3aeae151
Hide whitespace changes
Inline Side-by-side
Showing with 25 additions and 34 deletions
Cargo.lock
View file @ 07207a1b
...@@ -905,12 +905,12 @@ dependencies = [ ...@@ -905,12 +905,12 @@ dependencies = [
[[package]] [[package]]
name = "candle-core" name = "candle-core"
version = "0.9.1" version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=95d713f9#95d713f996225a3643b1d0c3eeb5a35c40516625" source = "git+https://github.com/EricLBuehler/candle.git?rev=7511e510#7511e510054973ea4e2043f6d2da14409c195baf"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"candle-kernels", "candle-kernels",
"candle-metal-kernels", "candle-metal-kernels",
"cudarc 0.16.6", "cudarc 0.17.3",
"float8", "float8",
"gemm 0.17.1", "gemm 0.17.1",
"half 2.6.0", "half 2.6.0",
...@@ -933,7 +933,7 @@ dependencies = [ ...@@ -933,7 +933,7 @@ dependencies = [
[[package]] [[package]]
name = "candle-kernels" name = "candle-kernels"
version = "0.9.1" version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=95d713f9#95d713f996225a3643b1d0c3eeb5a35c40516625" source = "git+https://github.com/EricLBuehler/candle.git?rev=7511e510#7511e510054973ea4e2043f6d2da14409c195baf"
dependencies = [ dependencies = [
"bindgen_cuda 0.1.5", "bindgen_cuda 0.1.5",
] ]
...@@ -941,7 +941,7 @@ dependencies = [ ...@@ -941,7 +941,7 @@ dependencies = [
[[package]] [[package]]
name = "candle-metal-kernels" name = "candle-metal-kernels"
version = "0.9.1" version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=95d713f9#95d713f996225a3643b1d0c3eeb5a35c40516625" source = "git+https://github.com/EricLBuehler/candle.git?rev=7511e510#7511e510054973ea4e2043f6d2da14409c195baf"
dependencies = [ dependencies = [
"half 2.6.0", "half 2.6.0",
"metal 0.27.0", "metal 0.27.0",
...@@ -953,9 +953,9 @@ dependencies = [ ...@@ -953,9 +953,9 @@ dependencies = [
[[package]] [[package]]
name = "candle-nn" name = "candle-nn"
version = "0.9.1" version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=95d713f9#95d713f996225a3643b1d0c3eeb5a35c40516625" source = "git+https://github.com/EricLBuehler/candle.git?rev=7511e510#7511e510054973ea4e2043f6d2da14409c195baf"
dependencies = [ dependencies = [
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=95d713f9)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)",
"candle-metal-kernels", "candle-metal-kernels",
"half 2.6.0", "half 2.6.0",
"metal 0.27.0", "metal 0.27.0",
...@@ -1611,6 +1611,7 @@ version = "0.17.3" ...@@ -1611,6 +1611,7 @@ version = "0.17.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ba848ae5c6f3cb36e71eab5f268763e3fabcabe3f7bc683e16f7fa3d46281e" checksum = "72ba848ae5c6f3cb36e71eab5f268763e3fabcabe3f7bc683e16f7fa3d46281e"
dependencies = [ dependencies = [
"half 2.6.0",
"libloading", "libloading",
] ]
...@@ -2703,11 +2704,11 @@ dependencies = [ ...@@ -2703,11 +2704,11 @@ dependencies = [
[[package]] [[package]]
name = "float8" name = "float8"
version = "0.3.0" version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f498aec3b227cd892ce18967f4033d9d397d28a80a7ab67e9f6b0176a79654e" checksum = "4203231de188ebbdfb85c11f3c20ca2b063945710de04e7b59268731e728b462"
dependencies = [ dependencies = [
"cudarc 0.16.6", "cudarc 0.17.3",
"half 2.6.0", "half 2.6.0",
"num-traits", "num-traits",
] ]
...@@ -4737,10 +4738,10 @@ dependencies = [ ...@@ -4737,10 +4738,10 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs" name = "mistralrs"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=53ebc74#53ebc7480cd08b124bc99ca9f6365af6152c6fa7" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=95d713f9)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)",
"candle-nn", "candle-nn",
"clap 4.5.48", "clap 4.5.48",
"either", "either",
...@@ -4759,7 +4760,7 @@ dependencies = [ ...@@ -4759,7 +4760,7 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-audio" name = "mistralrs-audio"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=53ebc74#53ebc7480cd08b124bc99ca9f6365af6152c6fa7" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"apodize", "apodize",
...@@ -4770,7 +4771,7 @@ dependencies = [ ...@@ -4770,7 +4771,7 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-core" name = "mistralrs-core"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=53ebc74#53ebc7480cd08b124bc99ca9f6365af6152c6fa7" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea"
dependencies = [ dependencies = [
"ahash", "ahash",
"akin", "akin",
...@@ -4783,7 +4784,7 @@ dependencies = [ ...@@ -4783,7 +4784,7 @@ dependencies = [
"bm25", "bm25",
"bytemuck", "bytemuck",
"bytemuck_derive", "bytemuck_derive",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=95d713f9)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)",
"candle-nn", "candle-nn",
"cfgrammar", "cfgrammar",
"chrono", "chrono",
...@@ -4866,7 +4867,7 @@ dependencies = [ ...@@ -4866,7 +4867,7 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-mcp" name = "mistralrs-mcp"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=53ebc74#53ebc7480cd08b124bc99ca9f6365af6152c6fa7" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-trait", "async-trait",
...@@ -4886,11 +4887,11 @@ dependencies = [ ...@@ -4886,11 +4887,11 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-paged-attn" name = "mistralrs-paged-attn"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=53ebc74#53ebc7480cd08b124bc99ca9f6365af6152c6fa7" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bindgen_cuda 0.1.7", "bindgen_cuda 0.1.7",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=95d713f9)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)",
"float8", "float8",
"half 2.6.0", "half 2.6.0",
"metal 0.27.0", "metal 0.27.0",
...@@ -4901,11 +4902,11 @@ dependencies = [ ...@@ -4901,11 +4902,11 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-quant" name = "mistralrs-quant"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=53ebc74#53ebc7480cd08b124bc99ca9f6365af6152c6fa7" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea"
dependencies = [ dependencies = [
"bindgen_cuda 0.1.7", "bindgen_cuda 0.1.7",
"byteorder", "byteorder",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=95d713f9)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)",
"candle-nn", "candle-nn",
"float8", "float8",
"half 2.6.0", "half 2.6.0",
...@@ -4929,9 +4930,9 @@ dependencies = [ ...@@ -4929,9 +4930,9 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-vision" name = "mistralrs-vision"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=53ebc74#53ebc7480cd08b124bc99ca9f6365af6152c6fa7" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea"
dependencies = [ dependencies = [
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=95d713f9)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)",
"image", "image",
"rayon", "rayon",
] ]
......
lib/engines/mistralrs/Cargo.toml
View file @ 07207a1b
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
[package] [package]
name = "dynamo-engine-mistralrs" name = "dynamo-engine-mistralrs"
...@@ -39,7 +27,7 @@ async-stream = { workspace = true } ...@@ -39,7 +27,7 @@ async-stream = { workspace = true }
async-trait = { workspace = true } async-trait = { workspace = true }
either = { workspace = true } either = { workspace = true }
indexmap = { version = "2.9.0", features = ["serde"] } indexmap = { version = "2.9.0", features = ["serde"] }
mistralrs = { git = "https://github.com/EricLBuehler/mistral.rs.git", version = "0.6.0", rev = "53ebc74" } mistralrs = { git = "https://github.com/EricLBuehler/mistral.rs.git", version = "0.6.0", rev = "a13220255" }
serde_json = { workspace = true } serde_json = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
lib/engines/mistralrs/src/lib.rs
View file @ 07207a1b
...@@ -312,6 +312,7 @@ impl ...@@ -312,6 +312,7 @@ impl
.unwrap_or(det.top_n_logprobs), .unwrap_or(det.top_n_logprobs),
frequency_penalty: request.inner.frequency_penalty.or(det.frequency_penalty), frequency_penalty: request.inner.frequency_penalty.or(det.frequency_penalty),
presence_penalty: request.inner.presence_penalty.or(det.presence_penalty), presence_penalty: request.inner.presence_penalty.or(det.presence_penalty),
repetition_penalty: det.repetition_penalty,
stop_toks: request.inner.stop.map(to_stop_tokens).or(det.stop_toks), stop_toks: request.inner.stop.map(to_stop_tokens).or(det.stop_toks),
max_len: { max_len: {
let requested_max_tokens = request let requested_max_tokens = request
...@@ -510,6 +511,7 @@ impl ...@@ -510,6 +511,7 @@ impl
.unwrap_or(det.top_n_logprobs), .unwrap_or(det.top_n_logprobs),
frequency_penalty: request.inner.frequency_penalty.or(det.frequency_penalty), frequency_penalty: request.inner.frequency_penalty.or(det.frequency_penalty),
presence_penalty: request.inner.presence_penalty.or(det.presence_penalty), presence_penalty: request.inner.presence_penalty.or(det.presence_penalty),
repetition_penalty: det.repetition_penalty,
stop_toks: request stop_toks: request
.inner .inner
.stop .stop
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment