[Model] Add starcoder2 awq support (#3569)

b0dfa91d · 少年 · GitHub · 56a8652f · b0dfa91d
Unverified Commit b0dfa91d authored Mar 25, 2024 by 少年 Committed by GitHub Mar 24, 2024
Show whitespace changes
Inline Side-by-side

Showing with 3 additions and 2 deletions

vllm/model_executor/models/starcoder2.py vllm/model_executor/models/starcoder2.py +3 -2

No files found.
--- a/vllm/model_executor/models/starcoder2.py
+++ b/vllm/model_executor/models/starcoder2.py
@@ -141,8 +141,9 @@ class Starcoder2MLP(nn.Module):
            bias=config.use_bias,
            linear_method=linear_method,
        )
-        self.act = get_act_fn(config.hidden_act,
-                              intermediate_size=config.intermediate_size)
+        quant_config = getattr(linear_method, "quant_config", None)
+        self.act = get_act_fn(config.hidden_act, quant_config,
+                              config.intermediate_size)

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states, _ = self.c_fc(hidden_states)