Merge remote-tracking branch 'origin/v0.15.1-dev' into v0.15.1-dev

# Conflicts:
#	vllm/model_executor/layers/fused_moe/config.py
#	vllm/model_executor/layers/fused_moe/layer.py
#	vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_marlin.py

vllm/model_executor/models/llava.py

@@ -662,7 +662,7 @@ class LlavaForConditionalGeneration(
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -853,4 +853,4 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
     dummy_inputs=LlavaDummyInputsBuilder,
 )
 class MantisForConditionalGeneration(LlavaForConditionalGeneration):
-    pass
+    pass
\ No newline at end of file

vllm/model_executor/models/llava_next.py

@@ -509,7 +509,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -577,4 +577,4 @@ model_executor.models.llava_next.LlavaNextProcessingInfo.get_num_image_tokens].
 
     def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         loader = AutoWeightsLoader(self)
-        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
\ No newline at end of file

vllm/model_executor/models/llava_next_video.py

@@ -426,7 +426,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -459,4 +459,4 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
             # This model doesn't support images for now
             ignore_unexpected_prefixes=["image_newline"],
         )
-        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
\ No newline at end of file

vllm/model_executor/models/llava_onevision.py

@@ -887,7 +887,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -916,4 +916,4 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
 
     def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         loader = AutoWeightsLoader(self)
-        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
\ No newline at end of file

vllm/model_executor/models/longcat_flash.py

@@ -520,7 +520,7 @@ class FlashModel(nn.Module):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -605,7 +605,7 @@ class LongcatFlashForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -764,4 +764,4 @@ class LongcatFlashForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
                     self_attn.kv_a_layernorm.weight.data *= (
                         self.config.hidden_size / self.config.kv_lora_rank
                     ) ** 0.5
-        return loaded_params
+        return loaded_params
\ No newline at end of file

vllm/model_executor/models/longcat_flash_mtp.py

@@ -150,7 +150,7 @@ class LongCatFlashMTP(nn.Module):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         hidden_states: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
@@ -345,4 +345,4 @@ class LongCatFlashMTP(nn.Module):
     ) -> int | None:
         if "model.mtp" in weight_name:
             return config.num_hidden_layers * 2
-        return None
+        return None
\ No newline at end of file

vllm/model_executor/models/mamba.py

@@ -142,7 +142,7 @@ class MambaModel(nn.Module):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -225,7 +225,7 @@ class MambaForCausalLM(
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -279,4 +279,4 @@ class MambaForCausalLM(
 
     def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         loader = AutoWeightsLoader(self)
-        return loader.load_weights(weights)
+        return loader.load_weights(weights)
\ No newline at end of file

vllm/model_executor/models/mamba2.py

@@ -137,7 +137,7 @@ class Mamba2Model(nn.Module):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -267,7 +267,7 @@ class Mamba2ForCausalLM(
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -291,4 +291,4 @@ class Mamba2ForCausalLM(
 
     def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         loader = AutoWeightsLoader(self)
-        return loader.load_weights(weights)
+        return loader.load_weights(weights)
\ No newline at end of file

vllm/model_executor/models/medusa.py

@@ -2,8 +2,6 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import os
-from typing import Iterable, List, Optional, Set, Tuple, Any, Dict
-
 from collections.abc import Iterable
 
 import torch
@@ -20,8 +18,6 @@ from .utils import maybe_prefix
 
 from vllm import _custom_ops as ops
 
-TOPK=10 # topk for sparse tree (10 is a placeholder and it is sufficient)
-
 
 class ResidualBlock(nn.Module):
     def __init__(self, config: VllmConfig, hidden_size: int, num_layers: int) -> None:

vllm/model_executor/models/midashenglm.py

@@ -796,7 +796,7 @@ class MiDashengLMModel(nn.Module, SupportsMultiModal, SupportsPP):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -820,4 +820,4 @@ class MiDashengLMModel(nn.Module, SupportsMultiModal, SupportsPP):
 
     def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         loader = AutoWeightsLoader(self)
-        return loader.load_weights(weights)
+        return loader.load_weights(weights)
\ No newline at end of file

vllm/model_executor/models/mimo.py

@@ -61,7 +61,7 @@ logger = init_logger(__name__)
 class MiMoModel(Qwen2Model):
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -185,4 +185,4 @@ class MiMoForCausalLM(Qwen2ForCausalLM, nn.Module):
     ) -> torch.Tensor | None:
         hidden_states = self.model.norm(hidden_states)
         logits = self.logits_processor(self.lm_head, hidden_states)
-        return logits
+        return logits
\ No newline at end of file

vllm/model_executor/models/mimo_mtp.py

@@ -169,7 +169,7 @@ class MiMoMTP(nn.Module):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         hidden_states: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
@@ -291,4 +291,4 @@ class MiMoMTP(nn.Module):
             name = name.replace(
                 f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block."
             )
-        return name
+        return name
\ No newline at end of file

vllm/model_executor/models/mimo_v2_flash.py

@@ -478,7 +478,7 @@ class MiMoV2Model(nn.Module):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -693,7 +693,7 @@ class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -715,4 +715,4 @@ class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
 
     def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         loader = AutoWeightsLoader(self)
-        return loader.load_weights(weights)
+        return loader.load_weights(weights)
\ No newline at end of file

vllm/model_executor/models/minicpm.py

@@ -440,7 +440,7 @@ class MiniCPMModel(nn.Module):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -620,7 +620,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -654,4 +654,4 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
             self,
             skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None),
         )
-        return loader.load_weights(weights)
+        return loader.load_weights(weights)
\ No newline at end of file

vllm/model_executor/models/minicpmv.py

@@ -1147,7 +1147,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -1740,4 +1740,4 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
         # so update values before init is called
         cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
         cls.embedding_modules.update(instance_cls.embedding_modules)
-        return instance_cls(vllm_config=vllm_config, prefix=prefix)
+        return instance_cls(vllm_config=vllm_config, prefix=prefix)
\ No newline at end of file

vllm/model_executor/models/minimax_m2.py

@@ -362,7 +362,7 @@ class MiniMaxM2Model(nn.Module):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None,
         inputs_embeds: torch.Tensor | None = None,
@@ -521,7 +521,7 @@ class MiniMaxM2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -555,4 +555,4 @@ def get_spec_layer_idx_from_weight_name(
         for i in range(config.num_mtp_modules):
             if weight_name.startswith(f"model.layers.{layer_idx + i}."):
                 return layer_idx + i
-    return None
+    return None
\ No newline at end of file

vllm/model_executor/models/minimax_text_01.py

@@ -712,7 +712,7 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid):
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -1011,4 +1011,4 @@ class MiniMaxText01ForCausalLM(nn.Module, HasInnerState, IsHybrid):
 
     @classmethod
     def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc]:
-        return MambaStateCopyFuncCalculator.linear_attention_state_copy_func()
+        return MambaStateCopyFuncCalculator.linear_attention_state_copy_func()
\ No newline at end of file

vllm/model_executor/models/minimax_vl_01.py

@@ -359,7 +359,7 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -382,4 +382,4 @@ class MiniMaxVL01ForConditionalGeneration(nn.Module, SupportsMultiModal, Support
 
     def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
         loader = AutoWeightsLoader(self)
-        return loader.load_weights(weights)
+        return loader.load_weights(weights)
\ No newline at end of file

vllm/model_executor/models/mistral.py

@@ -156,8 +156,16 @@ class MistralDecoderLayer(LlamaDecoderLayer):
         )
 
         self.layer_idx = int(prefix.split(sep=".")[-1])
+        quant_config = self.get_quant_config(vllm_config)
         config = config or vllm_config.model_config.hf_config
 
+        do_fusion = getattr(
+            quant_config, "enable_quantization_scaling_fusion", False
+        ) and vllm_config.cache_config.cache_dtype.startswith("fp8")
+        if do_fusion:
+            self.input_layernorm.quant_scaling_from = self.self_attn.qkv_proj
+            self.post_attention_layernorm.quant_scaling_from = self.mlp.gate_up_proj
+
         if getattr(config, "ada_rms_norm_t_cond", False):
             self.ada_rms_norm_t_cond = nn.Sequential(
                 ColumnParallelLinear(
@@ -339,4 +347,4 @@ class MistralForCausalLM(LlamaForCausalLM):
             elif item in mapping and mapping[item] not in name:
                 name = name.replace(item, mapping[item])
 
-        return name, loaded_weight
+        return name, loaded_weight
\ No newline at end of file

vllm/model_executor/models/mistral3.py

@@ -539,7 +539,7 @@ class Mistral3ForConditionalGeneration(
 
     def forward(
         self,
-        input_ids: torch.Tensor | None,
+        input_ids: torch.Tensor,
         positions: torch.Tensor,
         intermediate_tensors: IntermediateTensors | None = None,
         inputs_embeds: torch.Tensor | None = None,
@@ -609,4 +609,4 @@ class Mistral3ForConditionalGeneration(
             language_model="language_model",
             connector="multi_modal_projector",
             tower_model="vision_tower",
-        )
+        )
\ No newline at end of file