Skip to content

GitLab

"vscode:/vscode.git/clone" did not exist on "a1766e4a51e3568a25781c6f37672e2eafa82535"
Unverified Commit 2100f6aa authored by Graham King's avatar Graham King Committed by GitHub
Browse files

feat(mistralrs): Upgrade to support CUDA 13 (#4474) 


Signed-off-by: default avatarGraham King <grahamk@nvidia.com>
parent 17e22476
Expand all Show whitespace changes
Inline Side-by-side
Showing with 496 additions and 190 deletions
Cargo.lock
View file @ 2100f6aa
...@@ -1231,7 +1231,7 @@ dependencies = [ ...@@ -1231,7 +1231,7 @@ dependencies = [
"rayon", "rayon",
"safetensors 0.4.5", "safetensors 0.4.5",
"thiserror 1.0.69", "thiserror 1.0.69",
"ug", "ug 0.4.0",
"yoke 0.7.5", "yoke 0.7.5",
"zip 1.1.4", "zip 1.1.4",
] ]
...@@ -1239,12 +1239,12 @@ dependencies = [ ...@@ -1239,12 +1239,12 @@ dependencies = [
[[package]] [[package]]
name = "candle-core" name = "candle-core"
version = "0.9.1" version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=7511e510#7511e510054973ea4e2043f6d2da14409c195baf" source = "git+https://github.com/EricLBuehler/candle.git?rev=175926c9#175926c960f5aff6a543a06b3214471dd92785e4"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"candle-kernels", "candle-kernels",
"candle-metal-kernels", "candle-metal-kernels",
"cudarc 0.17.8", "cudarc",
"float8", "float8",
"gemm 0.17.1", "gemm 0.17.1",
"half 2.7.1", "half 2.7.1",
...@@ -1257,7 +1257,7 @@ dependencies = [ ...@@ -1257,7 +1257,7 @@ dependencies = [
"rayon", "rayon",
"safetensors 0.6.2", "safetensors 0.6.2",
"thiserror 1.0.69", "thiserror 1.0.69",
"ug", "ug 0.5.0",
"ug-cuda", "ug-cuda",
"ug-metal", "ug-metal",
"yoke 0.7.5", "yoke 0.7.5",
...@@ -1267,7 +1267,7 @@ dependencies = [ ...@@ -1267,7 +1267,7 @@ dependencies = [
[[package]] [[package]]
name = "candle-kernels" name = "candle-kernels"
version = "0.9.1" version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=7511e510#7511e510054973ea4e2043f6d2da14409c195baf" source = "git+https://github.com/EricLBuehler/candle.git?rev=175926c9#175926c960f5aff6a543a06b3214471dd92785e4"
dependencies = [ dependencies = [
"bindgen_cuda 0.1.5", "bindgen_cuda 0.1.5",
] ]
...@@ -1275,7 +1275,7 @@ dependencies = [ ...@@ -1275,7 +1275,7 @@ dependencies = [
[[package]] [[package]]
name = "candle-metal-kernels" name = "candle-metal-kernels"
version = "0.9.1" version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=7511e510#7511e510054973ea4e2043f6d2da14409c195baf" source = "git+https://github.com/EricLBuehler/candle.git?rev=175926c9#175926c960f5aff6a543a06b3214471dd92785e4"
dependencies = [ dependencies = [
"half 2.7.1", "half 2.7.1",
"metal 0.27.0", "metal 0.27.0",
...@@ -1287,9 +1287,9 @@ dependencies = [ ...@@ -1287,9 +1287,9 @@ dependencies = [
[[package]] [[package]]
name = "candle-nn" name = "candle-nn"
version = "0.9.1" version = "0.9.1"
source = "git+https://github.com/EricLBuehler/candle.git?rev=7511e510#7511e510054973ea4e2043f6d2da14409c195baf" source = "git+https://github.com/EricLBuehler/candle.git?rev=175926c9#175926c960f5aff6a543a06b3214471dd92785e4"
dependencies = [ dependencies = [
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"candle-metal-kernels", "candle-metal-kernels",
"half 2.7.1", "half 2.7.1",
"metal 0.27.0", "metal 0.27.0",
...@@ -2004,16 +2004,6 @@ dependencies = [ ...@@ -2004,16 +2004,6 @@ dependencies = [
"cipher", "cipher",
] ]
[[package]]
name = "cudarc"
version = "0.16.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17200eb07e7d85a243aa1bf4569a7aa998385ba98d14833973a817a63cc86e92"
dependencies = [
"half 2.7.1",
"libloading",
]
[[package]] [[package]]
name = "cudarc" name = "cudarc"
version = "0.17.8" version = "0.17.8"
...@@ -2447,7 +2437,7 @@ dependencies = [ ...@@ -2447,7 +2437,7 @@ dependencies = [
"libc", "libc",
"option-ext", "option-ext",
"redox_users", "redox_users",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
...@@ -2682,7 +2672,7 @@ dependencies = [ ...@@ -2682,7 +2672,7 @@ dependencies = [
"candle-core 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "candle-core 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono", "chrono",
"criterion 0.3.6", "criterion 0.3.6",
"cudarc 0.17.8", "cudarc",
"dashmap 5.5.3", "dashmap 5.5.3",
"derive-getters", "derive-getters",
"derive_builder", "derive_builder",
...@@ -2742,8 +2732,8 @@ dependencies = [ ...@@ -2742,8 +2732,8 @@ dependencies = [
"tokio-rayon", "tokio-rayon",
"tokio-stream", "tokio-stream",
"tokio-util", "tokio-util",
"toktrie 1.3.0", "toktrie 1.4.0",
"toktrie_hf_tokenizers 1.3.0", "toktrie_hf_tokenizers 1.4.0",
"tonic 0.13.1", "tonic 0.13.1",
"tonic-build 0.13.1", "tonic-build 0.13.1",
"tower 0.5.2", "tower 0.5.2",
...@@ -2764,7 +2754,7 @@ name = "dynamo-memory" ...@@ -2764,7 +2754,7 @@ name = "dynamo-memory"
version = "0.7.0" version = "0.7.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"cudarc 0.17.8", "cudarc",
"dynamo-config", "dynamo-config",
"libc", "libc",
"nix 0.30.1", "nix 0.30.1",
...@@ -3082,7 +3072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" ...@@ -3082,7 +3072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
...@@ -3308,7 +3298,7 @@ version = "0.4.2" ...@@ -3308,7 +3298,7 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4203231de188ebbdfb85c11f3c20ca2b063945710de04e7b59268731e728b462" checksum = "4203231de188ebbdfb85c11f3c20ca2b063945710de04e7b59268731e728b462"
dependencies = [ dependencies = [
"cudarc 0.17.8", "cudarc",
"half 2.7.1", "half 2.7.1",
"num-traits", "num-traits",
] ]
...@@ -4862,7 +4852,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" ...@@ -4862,7 +4852,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [ dependencies = [
"hermit-abi 0.5.2", "hermit-abi 0.5.2",
"libc", "libc",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
...@@ -4934,7 +4924,7 @@ dependencies = [ ...@@ -4934,7 +4924,7 @@ dependencies = [
"portable-atomic", "portable-atomic",
"portable-atomic-util", "portable-atomic-util",
"serde_core", "serde_core",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
...@@ -6083,10 +6073,10 @@ dependencies = [ ...@@ -6083,10 +6073,10 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs" name = "mistralrs"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"candle-nn", "candle-nn",
"clap 4.5.52", "clap 4.5.52",
"either", "either",
...@@ -6099,13 +6089,15 @@ dependencies = [ ...@@ -6099,13 +6089,15 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"tokio", "tokio",
"tracing",
"tracing-subscriber",
"walkdir", "walkdir",
] ]
[[package]] [[package]]
name = "mistralrs-audio" name = "mistralrs-audio"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"apodize", "apodize",
...@@ -6116,7 +6108,7 @@ dependencies = [ ...@@ -6116,7 +6108,7 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-core" name = "mistralrs-core"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [ dependencies = [
"ahash", "ahash",
"akin", "akin",
...@@ -6129,7 +6121,7 @@ dependencies = [ ...@@ -6129,7 +6121,7 @@ dependencies = [
"bm25", "bm25",
"bytemuck", "bytemuck",
"bytemuck_derive", "bytemuck_derive",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"candle-nn", "candle-nn",
"cfgrammar", "cfgrammar",
"chrono", "chrono",
...@@ -6166,7 +6158,6 @@ dependencies = [ ...@@ -6166,7 +6158,6 @@ dependencies = [
"mistralrs-vision", "mistralrs-vision",
"num-traits", "num-traits",
"objc", "objc",
"once_cell",
"ordered-float 5.1.0", "ordered-float 5.1.0",
"parking_lot", "parking_lot",
"radix_trie", "radix_trie",
...@@ -6212,7 +6203,7 @@ dependencies = [ ...@@ -6212,7 +6203,7 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-mcp" name = "mistralrs-mcp"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-trait", "async-trait",
...@@ -6232,26 +6223,25 @@ dependencies = [ ...@@ -6232,26 +6223,25 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-paged-attn" name = "mistralrs-paged-attn"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bindgen_cuda 0.1.7", "bindgen_cuda 0.1.7",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"float8", "float8",
"half 2.7.1", "half 2.7.1",
"metal 0.27.0", "metal 0.27.0",
"once_cell",
"thiserror 2.0.17", "thiserror 2.0.17",
] ]
[[package]] [[package]]
name = "mistralrs-quant" name = "mistralrs-quant"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [ dependencies = [
"bindgen_cuda 0.1.7", "bindgen_cuda 0.1.7",
"byteorder", "byteorder",
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"candle-nn", "candle-nn",
"float8", "float8",
"half 2.7.1", "half 2.7.1",
...@@ -6259,7 +6249,6 @@ dependencies = [ ...@@ -6259,7 +6249,6 @@ dependencies = [
"lazy_static", "lazy_static",
"memmap2", "memmap2",
"metal 0.27.0", "metal 0.27.0",
"once_cell",
"paste", "paste",
"rayon", "rayon",
"regex", "regex",
...@@ -6275,9 +6264,9 @@ dependencies = [ ...@@ -6275,9 +6264,9 @@ dependencies = [
[[package]] [[package]]
name = "mistralrs-vision" name = "mistralrs-vision"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=a13220255#a132202551f4fdf3a1d9e2da51232581789af7ea" source = "git+https://github.com/EricLBuehler/mistral.rs.git?rev=2bcf0e9e3#2bcf0e9e3a0c688786cfa3142e2b18844a7f3447"
dependencies = [ dependencies = [
"candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=7511e510)", "candle-core 0.9.1 (git+https://github.com/EricLBuehler/candle.git?rev=175926c9)",
"image", "image",
"rayon", "rayon",
] ]
...@@ -6801,7 +6790,7 @@ version = "0.50.3" ...@@ -6801,7 +6790,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [ dependencies = [
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
...@@ -8245,7 +8234,7 @@ dependencies = [ ...@@ -8245,7 +8234,7 @@ dependencies = [
"once_cell", "once_cell",
"socket2 0.6.1", "socket2 0.6.1",
"tracing", "tracing",
"windows-sys 0.59.0", "windows-sys 0.60.2",
] ]
[[package]] [[package]]
...@@ -8688,9 +8677,9 @@ dependencies = [ ...@@ -8688,9 +8677,9 @@ dependencies = [
[[package]] [[package]]
name = "resolv-conf" name = "resolv-conf"
version = "0.7.5" version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7"
[[package]] [[package]]
name = "rgb" name = "rgb"
...@@ -8972,7 +8961,7 @@ dependencies = [ ...@@ -8972,7 +8961,7 @@ dependencies = [
"errno", "errno",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
...@@ -10317,7 +10306,7 @@ dependencies = [ ...@@ -10317,7 +10306,7 @@ dependencies = [
"getrandom 0.3.4", "getrandom 0.3.4",
"once_cell", "once_cell",
"rustix", "rustix",
"windows-sys 0.59.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
...@@ -10725,9 +10714,9 @@ dependencies = [ ...@@ -10725,9 +10714,9 @@ dependencies = [
[[package]] [[package]]
name = "toktrie" name = "toktrie"
version = "1.3.0" version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55c5e672ee5e0311bbb37ac0f9f60ed7db38463365e6cf4f71240c7b15bb374d" checksum = "dcfbe778b5bf5ffda8e2c5e79540bf40505ce1fb48ba31e8b43337fe55ca4c23"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bytemuck", "bytemuck",
...@@ -10751,16 +10740,16 @@ dependencies = [ ...@@ -10751,16 +10740,16 @@ dependencies = [
[[package]] [[package]]
name = "toktrie_hf_tokenizers" name = "toktrie_hf_tokenizers"
version = "1.3.0" version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baec7fb81a716f85ae1e07ce951f9e9bc1b745635edd48cbebb6a033c6491dca" checksum = "2ff6e566c3ec913607f16076ae2619fb018c468ed4715b899ab99898c850934b"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"log", "log",
"serde", "serde",
"serde_json", "serde_json",
"tokenizers", "tokenizers",
"toktrie 1.3.0", "toktrie 1.4.0",
] ]
[[package]] [[package]]
...@@ -11263,31 +11252,52 @@ dependencies = [ ...@@ -11263,31 +11252,52 @@ dependencies = [
"yoke 0.7.5", "yoke 0.7.5",
] ]
[[package]]
name = "ug"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76b761acf8af3494640d826a8609e2265e19778fb43306c7f15379c78c9b05b0"
dependencies = [
"gemm 0.18.2",
"half 2.7.1",
"libloading",
"memmap2",
"num",
"num-traits",
"num_cpus",
"rayon",
"safetensors 0.4.5",
"serde",
"thiserror 1.0.69",
"tracing",
"yoke 0.7.5",
]
[[package]] [[package]]
name = "ug-cuda" name = "ug-cuda"
version = "0.4.0" version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14053653d0b7fa7b21015aa9a62edc8af2f60aa6f9c54e66386ecce55f22ed29" checksum = "9f0a1fa748f26166778c33b8498255ebb7c6bffb472bcc0a72839e07ebb1d9b5"
dependencies = [ dependencies = [
"cudarc 0.16.6", "cudarc",
"half 2.7.1", "half 2.7.1",
"serde", "serde",
"thiserror 1.0.69", "thiserror 1.0.69",
"ug", "ug 0.5.0",
] ]
[[package]] [[package]]
name = "ug-metal" name = "ug-metal"
version = "0.4.0" version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76daec3c7a32a1b4a0e3307b6b057fa067aa64e750713987410a2c402e5cd731" checksum = "9f7adf545a99a086d362efc739e7cf4317c18cbeda22706000fd434d70ea3d95"
dependencies = [ dependencies = [
"half 2.7.1", "half 2.7.1",
"metal 0.29.0", "metal 0.29.0",
"objc", "objc",
"serde", "serde",
"thiserror 1.0.69", "thiserror 1.0.69",
"ug", "ug 0.5.0",
] ]
[[package]] [[package]]
...@@ -12002,7 +12012,7 @@ version = "0.1.11" ...@@ -12002,7 +12012,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [ dependencies = [
"windows-sys 0.48.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
......
lib/engines/mistralrs/Cargo.toml
View file @ 2100f6aa
...@@ -27,7 +27,7 @@ async-stream = { workspace = true } ...@@ -27,7 +27,7 @@ async-stream = { workspace = true }
async-trait = { workspace = true } async-trait = { workspace = true }
either = { workspace = true } either = { workspace = true }
indexmap = { version = "2.9.0", features = ["serde"] } indexmap = { version = "2.9.0", features = ["serde"] }
mistralrs = { git = "https://github.com/EricLBuehler/mistral.rs.git", version = "0.6.0", rev = "a13220255" } mistralrs = { git = "https://github.com/EricLBuehler/mistral.rs.git", version = "0.6.0", rev = "2bcf0e9e3" }
serde_json = { workspace = true } serde_json = { workspace = true }
tokio = { workspace = true } tokio = { workspace = true }
tracing = { workspace = true } tracing = { workspace = true }
lib/engines/mistralrs/src/lib.rs
View file @ 2100f6aa
...@@ -136,7 +136,6 @@ impl MistralRsEngine { ...@@ -136,7 +136,6 @@ impl MistralRsEngine {
let paged_attention_config = if cfg!(feature = "cuda") && EXP_ENABLE_PAGED_ATTENTION { let paged_attention_config = if cfg!(feature = "cuda") && EXP_ENABLE_PAGED_ATTENTION {
Some(PagedAttentionConfig::new( Some(PagedAttentionConfig::new(
None, // Block size, default 32 None, // Block size, default 32
4096, // CPU memory in MiB
MemoryGpuConfig::ContextSize(max_seq_len), MemoryGpuConfig::ContextSize(max_seq_len),
PagedCacheType::Auto, PagedCacheType::Auto,
)?) )?)
...@@ -237,6 +236,7 @@ impl MistralRsEngine { ...@@ -237,6 +236,7 @@ impl MistralRsEngine {
logits_processors: None, logits_processors: None,
return_raw_logits: false, return_raw_logits: false,
web_search_options: None, web_search_options: None,
truncate_sequence: false,
})); }));
// Send warmup request and consume response // Send warmup request and consume response
...@@ -359,6 +359,7 @@ impl ...@@ -359,6 +359,7 @@ impl
logits_processors: None, logits_processors: None,
return_raw_logits: false, return_raw_logits: false,
web_search_options: None, web_search_options: None,
truncate_sequence: false,
})); }));
self.mistralrs self.mistralrs
...@@ -559,6 +560,7 @@ impl ...@@ -559,6 +560,7 @@ impl
logits_processors: None, logits_processors: None,
return_raw_logits: false, return_raw_logits: false,
web_search_options: None, web_search_options: None,
truncate_sequence: false,
})); }));
self.mistralrs self.mistralrs
......
lib/runtime/examples/Cargo.lock
View file @ 2100f6aa
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment