merge v0.3.1

7e1d5e53 · zhuwenwen · e3378b20 · 5f08050d · 7e1d5e53 · 7e1d5e53
Commit 7e1d5e53 authored Feb 19, 2024 by zhuwenwen
20 changed files
--- a/vllm/engine/metrics.py
+++ b/vllm/engine/metrics.py
--- a/vllm/engine/ray_utils.py
+++ b/vllm/engine/ray_utils.py
--- a/vllm/entrypoints/llm.py
+++ b/vllm/entrypoints/llm.py
@@ -111,13 +111,13 @@ class LLM:
    def get_tokenizer(
            self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
-        return self.llm_engine.tokenizer
+        return self.llm_engine.tokenizer.tokenizer
    def set_tokenizer(
        self,
        tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
    ) -> None:
-        self.llm_engine.tokenizer = tokenizer
+        self.llm_engine.tokenizer.tokenizer = tokenizer
    def generate(
        self,

--- a/vllm/entrypoints/openai/serving_completion.py
+++ b/vllm/entrypoints/openai/serving_completion.py
--- a/vllm/logger.py
+++ b/vllm/logger.py
--- a/vllm/lora/models.py
+++ b/vllm/lora/models.py
--- a/vllm/lora/punica.py
+++ b/vllm/lora/punica.py
--- a/vllm/lora/worker_manager.py
+++ b/vllm/lora/worker_manager.py
--- a/vllm/model_executor/layers/activation.py
+++ b/vllm/model_executor/layers/activation.py
@@ -89,9 +89,7 @@ class ScaledActivation(nn.Module):
        if params_dtype is None:
            params_dtype = torch.get_default_dtype()
        self.scales = nn.Parameter(
-            torch.empty(intermediate_size_per_partition,
+            torch.empty(intermediate_size_per_partition, dtype=params_dtype))
-                        dtype=params_dtype,
-                        device="cuda"))
        set_weight_attrs(self.scales, {"weight_loader": self.weight_loader})
    def forward(self, x: torch.Tensor) -> torch.Tensor:

--- a/vllm/model_executor/layers/attention.py
+++ b/vllm/model_executor/layers/attention.py
--- a/vllm/model_executor/layers/fused_moe.py
+++ b/vllm/model_executor/layers/fused_moe.py
--- a/vllm/model_executor/layers/linear.py
+++ b/vllm/model_executor/layers/linear.py
--- a/vllm/model_executor/layers/quantization/awq.py
+++ b/vllm/model_executor/layers/quantization/awq.py
--- a/vllm/model_executor/layers/quantization/gptq.py
+++ b/vllm/model_executor/layers/quantization/gptq.py
--- a/vllm/model_executor/layers/quantization/squeezellm.py
+++ b/vllm/model_executor/layers/quantization/squeezellm.py
--- a/vllm/model_executor/layers/rotary_embedding.py
+++ b/vllm/model_executor/layers/rotary_embedding.py
--- a/vllm/model_executor/layers/triton_kernel/prefix_prefill.py
+++ b/vllm/model_executor/layers/triton_kernel/prefix_prefill.py
--- a/vllm/model_executor/layers/vocab_parallel_embedding.py
+++ b/vllm/model_executor/layers/vocab_parallel_embedding.py
--- a/vllm/model_executor/model_loader.py
+++ b/vllm/model_executor/model_loader.py
--- a/vllm/model_executor/models/__init__.py
+++ b/vllm/model_executor/models/__init__.py