Merge tag 'v0.7.2' into v0.7.2-dev

vllm/model_executor/layers/quantization/kernels/scaled_mm/__init__.py

+# SPDX-License-Identifier: Apache-2.0
+
 import os
 from typing import Dict, List, Optional, Type
 

vllm/model_executor/layers/quantization/kernels/scaled_mm/cutlass.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Optional, Tuple
 
 import torch

vllm/model_executor/layers/quantization/kernels/scaled_mm/triton.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Optional, Tuple
 
 import torch

vllm/model_executor/layers/quantization/kernels/scaled_mm/xla.py

+# SPDX-License-Identifier: Apache-2.0
+
 import warnings
 from typing import Optional, Tuple
 

vllm/model_executor/layers/quantization/kv_cache.py

+# SPDX-License-Identifier: Apache-2.0
+
 import torch
 
 from vllm.logger import init_logger

vllm/model_executor/layers/quantization/marlin.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Any, Dict, List, Optional
 
 import torch

vllm/model_executor/layers/quantization/modelopt.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Any, Dict, List, Optional
 
 import torch

vllm/model_executor/layers/quantization/moe_wna16.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Any, Callable, Dict, List, Optional
 
 import torch
@@ -5,17 +7,15 @@ import torch
 from vllm.distributed import get_tensor_model_parallel_rank, get_tp_group
 from vllm.model_executor.layers.fused_moe.layer import (
     FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported)
-from vllm.model_executor.layers.linear import UnquantizedLinearMethod
-from vllm.model_executor.layers.quantization.awq import (AWQConfig,
-                                                         AWQLinearMethod)
-from vllm.model_executor.layers.quantization.awq_marlin import (
-    AWQMarlinConfig, AWQMarlinLinearMethod)
+from vllm.model_executor.layers.linear import (LinearBase,
+                                               UnquantizedLinearMethod)
+from vllm.model_executor.layers.quantization.awq import AWQConfig
+from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
 from vllm.model_executor.layers.quantization.base_config import (
     QuantizationConfig, QuantizeMethodBase)
-from vllm.model_executor.layers.quantization.gptq import (GPTQConfig,
-                                                          GPTQLinearMethod)
+from vllm.model_executor.layers.quantization.gptq import GPTQConfig
 from vllm.model_executor.layers.quantization.gptq_marlin import (
-    GPTQMarlinConfig, GPTQMarlinLinearMethod)
+    GPTQMarlinConfig)
 from vllm.model_executor.utils import set_weight_attrs
 from vllm.platforms import current_platform
 
@@ -126,25 +126,26 @@ class MoeWNA16Config(QuantizationConfig):
                          prefix: str) -> Optional["QuantizeMethodBase"]:
         if is_layer_skipped_quant(prefix, self.modules_to_not_convert):
             return UnquantizedLinearMethod()
-        elif isinstance(layer, FusedMoE):
-            return MoeWNA16Method(self)
-        else:
+        elif isinstance(layer, LinearBase):
             if self.linear_quant_method == "gptq":
                 if self.use_marlin:
-                    return GPTQMarlinLinearMethod(
-                        GPTQMarlinConfig.from_config(self.full_config))
+                    return GPTQMarlinConfig.from_config(
+                        self.full_config).get_quant_method(layer, prefix)
                 else:
-                    return GPTQLinearMethod(
-                        GPTQConfig.from_config(self.full_config))
+                    return GPTQConfig.from_config(
+                        self.full_config).get_quant_method(layer, prefix)
             elif self.linear_quant_method == "awq":
                 if self.use_marlin:
-                    return AWQMarlinLinearMethod(
-                        AWQMarlinConfig.from_config(self.full_config))
+                    return AWQMarlinConfig.from_config(
+                        self.full_config).get_quant_method(layer, prefix)
                 else:
-                    return AWQLinearMethod(
-                        AWQConfig.from_config(self.full_config))
+                    return AWQConfig.from_config(
+                        self.full_config).get_quant_method(layer, prefix)
             else:
                 raise ValueError("moe_wna16 only support gptq and awq.")
+        elif isinstance(layer, FusedMoE):
+            return MoeWNA16Method(self)
+        return None
 
 
 def is_layer_skipped_quant(prefix: str, modules_to_not_convert: List[str]):

vllm/model_executor/layers/quantization/neuron_quant.py

+# SPDX-License-Identifier: Apache-2.0
+
 import os
 from importlib.util import find_spec
 from typing import Any, Dict, List, Optional

vllm/model_executor/layers/quantization/qqq.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Any, Dict, List, Optional
 
 import torch

vllm/model_executor/layers/quantization/quark/quark.py

+# SPDX-License-Identifier: Apache-2.0
+
 import fnmatch
 import re
 from typing import Any, Dict, List, Optional, cast
@@ -16,8 +18,6 @@ from vllm.model_executor.layers.quantization.quark.schemes import (
     QuarkScheme, QuarkW8A8Fp8, QuarkW8A8Int8)
 from vllm.model_executor.layers.quantization.quark.utils import (
     deep_compare, should_ignore_layer)
-from vllm.model_executor.layers.quantization.utils.quant_utils import (
-    FUSED_LAYER_NAME_MAPPING)
 from vllm.platforms import current_platform
 
 __all__ = ["QuarkLinearMethod"]
@@ -56,7 +56,9 @@ class QuarkConfig(QuantizationConfig):
 
         # Check if the layer is skipped for quantization.
         exclude_layers = cast(List[str], self.quant_config.get("exclude"))
-        if should_ignore_layer(prefix, ignore=exclude_layers):
+        if should_ignore_layer(prefix,
+                               ignore=exclude_layers,
+                               fused_mapping=self.packed_modules_mapping):
             return UnquantizedLinearMethod()
         if isinstance(layer, LinearBase):
             scheme = self.get_scheme(layer=layer, layer_name=prefix)
@@ -199,8 +201,8 @@ class QuarkConfig(QuantizationConfig):
                              module: torch.nn.Module) -> Dict[str, Any]:
 
         proj_name = layer_name.split(".")[-1]
-        if proj_name in FUSED_LAYER_NAME_MAPPING:
-            shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
+        if proj_name in self.packed_modules_mapping:
+            shard_proj_names = self.packed_modules_mapping[proj_name]
 
             # Convert fused_name --> [shard_names]
             shard_names = [

vllm/model_executor/layers/quantization/quark/quark_moe.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Any, Callable, Dict, Optional
 
 import torch

vllm/model_executor/layers/quantization/quark/schemes/__init__.py

+# SPDX-License-Identifier: Apache-2.0
+
 from .quark_scheme import QuarkScheme
 from .quark_w8a8_fp8 import QuarkW8A8Fp8
 from .quark_w8a8_int8 import QuarkW8A8Int8

vllm/model_executor/layers/quantization/quark/schemes/quark_scheme.py

+# SPDX-License-Identifier: Apache-2.0
+
 from abc import ABC, abstractmethod
 from typing import Optional
 

vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_fp8.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Callable, List, Optional
 
 import torch

vllm/model_executor/layers/quantization/quark/schemes/quark_w8a8_int8.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Callable, List, Optional, Set
 
 import torch

vllm/model_executor/layers/quantization/quark/utils.py

-import re
-from typing import Any, Iterable, Optional
+# SPDX-License-Identifier: Apache-2.0
 
-from vllm.model_executor.layers.quantization.utils.quant_utils import (
-    FUSED_LAYER_NAME_MAPPING)
+import re
+from types import MappingProxyType
+from typing import Any, Iterable, List, Mapping, Optional
 
 
 def deep_compare(dict1: Any, dict2: Any) -> bool:
@@ -18,8 +18,11 @@ def deep_compare(dict1: Any, dict2: Any) -> bool:
         return dict1 == dict2
 
 
-def should_ignore_layer(layer_name: Optional[str],
-                        ignore: Iterable[str]) -> bool:
+def should_ignore_layer(
+    layer_name: Optional[str],
+    ignore: Iterable[str],
+    fused_mapping: Mapping[str, List[str]] = MappingProxyType({})
+) -> bool:
     if layer_name is None:
         return False
 
@@ -31,8 +34,8 @@ def should_ignore_layer(layer_name: Optional[str],
     # in the safetensors checkpoint. So, we convert the name
     # from the fused version to unfused + check to make sure that
     # each shard of the fused layer has the same scheme.
-    if proj_name in FUSED_LAYER_NAME_MAPPING:
-        shard_proj_names = FUSED_LAYER_NAME_MAPPING[proj_name]
+    if proj_name in fused_mapping:
+        shard_proj_names = fused_mapping[proj_name]
 
         # Convert fused_name --> [shard_names]
         shard_names = [

vllm/model_executor/layers/quantization/schema.py

+# SPDX-License-Identifier: Apache-2.0
 """
 This file contains the Pydantic schemas for various quantization-related
 parameters. When a relevant quantization technique is specified, these

vllm/model_executor/layers/quantization/tpu_int8.py

+# SPDX-License-Identifier: Apache-2.0
+
 from typing import Any, Dict, List, Optional, Tuple
 
 import torch

vllm/model_executor/layers/quantization/utils/__init__.py

+# SPDX-License-Identifier: Apache-2.0
+
 from .layer_utils import replace_parameter, update_tensor_inplace
 
 __all__ = ['update_tensor_inplace', 'replace_parameter']