Use flashinfer for Gemma 2.

ce7e3565 · Daniël de Kok · cf04a43f · ce7e3565
Commit ce7e3565 authored Oct 15, 2024 by Daniël de Kok
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

launcher/src/main.rs launcher/src/main.rs +1 -1

No files found.
--- a/launcher/src/main.rs
+++ b/launcher/src/main.rs
@@ -94,7 +94,7 @@ fn resolve_attention(config: &Option<Config>, lora_adapters: &Option<String>) ->
                    prefix_caching = Some("0".to_string());
                }
                match config.model_type.as_deref() {
-                    Some("gemma2") | Some("falcon") | Some("deepseek_v2") => {
+                    Some("falcon") | Some("deepseek_v2") => {
                        // Required because gemma2 needs bfloat16 which is not supported by
                        // flashinfer ?
                        if attention.is_none() {