Unverified Commit 4e256cad authored by Harry Mellor's avatar Harry Mellor Committed by GitHub
Browse files

Remove all references to `yapf` as it's no longer used (#26251)


Signed-off-by: default avatarHarry Mellor <19981378+hmellor@users.noreply.github.com>
parent d6953beb
...@@ -13,9 +13,6 @@ from fastapi import Request ...@@ -13,9 +13,6 @@ from fastapi import Request
from vllm.config import ModelConfig from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.logger import RequestLogger
# yapf conflicts with isort for this block
# yapf: disable
from vllm.entrypoints.openai.protocol import ( from vllm.entrypoints.openai.protocol import (
CompletionLogProbs, CompletionLogProbs,
CompletionRequest, CompletionRequest,
...@@ -29,8 +26,6 @@ from vllm.entrypoints.openai.protocol import ( ...@@ -29,8 +26,6 @@ from vllm.entrypoints.openai.protocol import (
UsageInfo, UsageInfo,
) )
from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs from vllm.entrypoints.openai.serving_engine import OpenAIServing, clamp_prompt_logprobs
# yapf: enable
from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.renderer import RenderConfig from vllm.entrypoints.renderer import RenderConfig
from vllm.entrypoints.utils import get_max_tokens from vllm.entrypoints.utils import get_max_tokens
......
...@@ -14,9 +14,6 @@ from vllm.config import ModelConfig ...@@ -14,9 +14,6 @@ from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.logger import RequestLogger
# yapf conflicts with isort for this docstring
# yapf: disable
from vllm.entrypoints.openai.protocol import ( from vllm.entrypoints.openai.protocol import (
EmbeddingChatRequest, EmbeddingChatRequest,
EmbeddingCompletionRequest, EmbeddingCompletionRequest,
...@@ -32,8 +29,6 @@ from vllm.entrypoints.openai.serving_engine import ( ...@@ -32,8 +29,6 @@ from vllm.entrypoints.openai.serving_engine import (
ServeContext, ServeContext,
TextTokensPrompt, TextTokensPrompt,
) )
# yapf: enable
from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.renderer import RenderConfig from vllm.entrypoints.renderer import RenderConfig
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
......
...@@ -28,9 +28,6 @@ else: ...@@ -28,9 +28,6 @@ else:
import vllm.envs as envs import vllm.envs as envs
from vllm.config import ModelConfig from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
# yapf conflicts with isort for this block
# yapf: disable
from vllm.entrypoints.chat_utils import ( from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam, ChatCompletionMessageParam,
ChatTemplateContentFormatOption, ChatTemplateContentFormatOption,
...@@ -72,8 +69,6 @@ from vllm.entrypoints.openai.protocol import ( ...@@ -72,8 +69,6 @@ from vllm.entrypoints.openai.protocol import (
from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.openai.tool_parsers import ToolParser from vllm.entrypoints.openai.tool_parsers import ToolParser
from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
# yapf: enable
from vllm.inputs.data import PromptType from vllm.inputs.data import PromptType
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
from vllm.inputs.parse import PromptComponents, get_prompt_components from vllm.inputs.parse import PromptComponents, get_prompt_components
......
...@@ -17,8 +17,6 @@ from vllm.config import VllmConfig ...@@ -17,8 +17,6 @@ from vllm.config import VllmConfig
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.logger import RequestLogger
# yapf: disable
from vllm.entrypoints.openai.protocol import ( from vllm.entrypoints.openai.protocol import (
ErrorResponse, ErrorResponse,
IOProcessorRequest, IOProcessorRequest,
...@@ -30,8 +28,6 @@ from vllm.entrypoints.openai.protocol import ( ...@@ -30,8 +28,6 @@ from vllm.entrypoints.openai.protocol import (
PoolingResponseData, PoolingResponseData,
UsageInfo, UsageInfo,
) )
# yapf: enable
from vllm.entrypoints.openai.serving_engine import OpenAIServing from vllm.entrypoints.openai.serving_engine import OpenAIServing
from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.renderer import RenderConfig from vllm.entrypoints.renderer import RenderConfig
......
...@@ -14,9 +14,6 @@ from typing import Callable, Final, Optional, Union ...@@ -14,9 +14,6 @@ from typing import Callable, Final, Optional, Union
import jinja2 import jinja2
from fastapi import Request from fastapi import Request
# yapf conflicts with isort for this block
# yapf: disable
from openai.types.responses import ( from openai.types.responses import (
ResponseCodeInterpreterCallCodeDeltaEvent, ResponseCodeInterpreterCallCodeDeltaEvent,
ResponseCodeInterpreterCallCodeDoneEvent, ResponseCodeInterpreterCallCodeDoneEvent,
...@@ -46,8 +43,6 @@ from openai.types.responses import ( ...@@ -46,8 +43,6 @@ from openai.types.responses import (
response_text_delta_event, response_text_delta_event,
) )
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
# yapf: enable
from openai.types.responses.response_reasoning_item import ( from openai.types.responses.response_reasoning_item import (
Content as ResponseReasoningTextContent, Content as ResponseReasoningTextContent,
) )
...@@ -78,9 +73,6 @@ from vllm.entrypoints.harmony_utils import ( ...@@ -78,9 +73,6 @@ from vllm.entrypoints.harmony_utils import (
render_for_completion, render_for_completion,
) )
from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.logger import RequestLogger
# yapf conflicts with isort for this block
# yapf: disable
from vllm.entrypoints.openai.protocol import ( from vllm.entrypoints.openai.protocol import (
DeltaMessage, DeltaMessage,
ErrorResponse, ErrorResponse,
...@@ -97,8 +89,6 @@ from vllm.entrypoints.openai.protocol import ( ...@@ -97,8 +89,6 @@ from vllm.entrypoints.openai.protocol import (
ResponseUsage, ResponseUsage,
StreamingResponsesResponse, StreamingResponsesResponse,
) )
# yapf: enable
from vllm.entrypoints.openai.serving_engine import OpenAIServing from vllm.entrypoints.openai.serving_engine import OpenAIServing
from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.tool_server import ToolServer from vllm.entrypoints.tool_server import ToolServer
......
...@@ -24,9 +24,6 @@ from vllm.entrypoints.openai.protocol import ( ...@@ -24,9 +24,6 @@ from vllm.entrypoints.openai.protocol import (
) )
from vllm.entrypoints.openai.serving_engine import OpenAIServing from vllm.entrypoints.openai.serving_engine import OpenAIServing
from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.openai.serving_models import OpenAIServingModels
# yapf conflicts with isort for this block
# yapf: disable
from vllm.entrypoints.score_utils import ( from vllm.entrypoints.score_utils import (
ScoreContentPartParam, ScoreContentPartParam,
ScoreMultiModalParam, ScoreMultiModalParam,
...@@ -35,8 +32,6 @@ from vllm.entrypoints.score_utils import ( ...@@ -35,8 +32,6 @@ from vllm.entrypoints.score_utils import (
compress_token_type_ids, compress_token_type_ids,
get_score_prompt, get_score_prompt,
) )
# yapf: enable
from vllm.entrypoints.utils import _validate_truncation_size from vllm.entrypoints.utils import _validate_truncation_size
from vllm.inputs.data import TokensPrompt from vllm.inputs.data import TokensPrompt
from vllm.logger import init_logger from vllm.logger import init_logger
......
...@@ -10,9 +10,6 @@ from vllm.config import ModelConfig ...@@ -10,9 +10,6 @@ from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.logger import RequestLogger
# yapf conflicts with isort for this block
# yapf: disable
from vllm.entrypoints.openai.protocol import ( from vllm.entrypoints.openai.protocol import (
DetokenizeRequest, DetokenizeRequest,
DetokenizeResponse, DetokenizeResponse,
...@@ -22,8 +19,6 @@ from vllm.entrypoints.openai.protocol import ( ...@@ -22,8 +19,6 @@ from vllm.entrypoints.openai.protocol import (
TokenizeResponse, TokenizeResponse,
TokenizerInfoResponse, TokenizerInfoResponse,
) )
# yapf: enable
from vllm.entrypoints.openai.serving_engine import OpenAIServing from vllm.entrypoints.openai.serving_engine import OpenAIServing
from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.openai.serving_models import OpenAIServingModels
from vllm.entrypoints.renderer import RenderConfig from vllm.entrypoints.renderer import RenderConfig
......
...@@ -11,7 +11,7 @@ import cloudpickle ...@@ -11,7 +11,7 @@ import cloudpickle
import msgspec import msgspec
import vllm.envs as envs import vllm.envs as envs
from vllm.executor.executor_base import DistributedExecutorBase # yapf: disable from vllm.executor.executor_base import DistributedExecutorBase
from vllm.executor.msgspec_utils import encode_hook from vllm.executor.msgspec_utils import encode_hook
from vllm.executor.ray_utils import RayWorkerWrapper, initialize_ray_cluster, ray from vllm.executor.ray_utils import RayWorkerWrapper, initialize_ray_cluster, ray
from vllm.logger import init_logger from vllm.logger import init_logger
......
...@@ -8,8 +8,6 @@ from transformers import PretrainedConfig ...@@ -8,8 +8,6 @@ from transformers import PretrainedConfig
from vllm.config.lora import LoRAConfig from vllm.config.lora import LoRAConfig
from vllm.distributed.utils import divide from vllm.distributed.utils import divide
# yapf: disable
from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.linear import (
ColumnParallelLinear, ColumnParallelLinear,
LinearBase, LinearBase,
...@@ -23,7 +21,6 @@ from .utils import _get_lora_device ...@@ -23,7 +21,6 @@ from .utils import _get_lora_device
class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
def __init__(self, base_layer: LinearBase): def __init__(self, base_layer: LinearBase):
super().__init__() super().__init__()
self.base_layer = base_layer self.base_layer = base_layer
...@@ -50,16 +47,20 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): ...@@ -50,16 +47,20 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
lora_b_out_size = self.output_size lora_b_out_size = self.output_size
elif isinstance(self.base_layer, ColumnParallelLinear): elif isinstance(self.base_layer, ColumnParallelLinear):
lora_a_out_size = (lora_config.max_lora_rank if lora_a_out_size = (
not lora_config.fully_sharded_loras else divide( lora_config.max_lora_rank
lora_config.max_lora_rank, self.tp_size)) if not lora_config.fully_sharded_loras
else divide(lora_config.max_lora_rank, self.tp_size)
)
lora_b_out_size = self.output_size lora_b_out_size = self.output_size
elif isinstance(self.base_layer, RowParallelLinear): elif isinstance(self.base_layer, RowParallelLinear):
lora_a_out_size = lora_config.max_lora_rank lora_a_out_size = lora_config.max_lora_rank
lora_b_out_size = (self.output_size if lora_b_out_size = (
not lora_config.fully_sharded_loras else divide( self.output_size
self.output_size, self.tp_size)) if not lora_config.fully_sharded_loras
else divide(self.output_size, self.tp_size)
)
else: else:
raise NotImplementedError raise NotImplementedError
...@@ -71,7 +72,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): ...@@ -71,7 +72,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
self.input_size, self.input_size,
dtype=lora_config.lora_dtype, dtype=lora_config.lora_dtype,
device=self.device, device=self.device,
) for _ in range(self.n_slices)) )
for _ in range(self.n_slices)
)
self.lora_b_stacked = tuple( self.lora_b_stacked = tuple(
torch.zeros( torch.zeros(
max_loras, max_loras,
...@@ -80,7 +83,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): ...@@ -80,7 +83,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
lora_config.max_lora_rank, lora_config.max_lora_rank,
dtype=lora_config.lora_dtype, dtype=lora_config.lora_dtype,
device=self.device, device=self.device,
) for _ in range(self.n_slices)) )
for _ in range(self.n_slices)
)
if lora_config.bias_enabled: if lora_config.bias_enabled:
lora_bias_out_size = lora_b_out_size lora_bias_out_size = lora_b_out_size
self.lora_bias_stacked = tuple( self.lora_bias_stacked = tuple(
...@@ -90,8 +95,10 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): ...@@ -90,8 +95,10 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
lora_bias_out_size, lora_bias_out_size,
dtype=lora_config.lora_dtype, dtype=lora_config.lora_dtype,
device=self.device, device=self.device,
) for _ in range(self.n_slices)) )
self.output_slices = (self.lora_b_stacked[0].shape[2], ) for _ in range(self.n_slices)
)
self.output_slices = (self.lora_b_stacked[0].shape[2],)
def reset_lora(self, index: int): def reset_lora(self, index: int):
for s_index in range(self.n_slices): for s_index in range(self.n_slices):
...@@ -99,8 +106,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): ...@@ -99,8 +106,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
self.lora_b_stacked[s_index][index] = 0 self.lora_b_stacked[s_index][index] = 0
if self.lora_config.bias_enabled: if self.lora_config.bias_enabled:
# Make mypy happy # Make mypy happy
self.lora_bias_stacked = cast(tuple[torch.Tensor, ...], self.lora_bias_stacked = cast(
self.lora_bias_stacked) tuple[torch.Tensor, ...], self.lora_bias_stacked
)
self.lora_bias_stacked[s_index][index] = 0 self.lora_bias_stacked[s_index][index] = 0
def set_lora( def set_lora(
...@@ -115,8 +123,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): ...@@ -115,8 +123,9 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
# MergedColumnParallelLinearWithLoRA, all other linear LoRA layers # MergedColumnParallelLinearWithLoRA, all other linear LoRA layers
# store weights in a tuple of size 1. These two layers will # store weights in a tuple of size 1. These two layers will
# override this function. # override this function.
assert (len(self.lora_a_stacked) == len(self.lora_b_stacked) == assert (
self.n_slices == 1) len(self.lora_a_stacked) == len(self.lora_b_stacked) == self.n_slices == 1
)
self.reset_lora(index) self.reset_lora(index)
if self.tp_size > 1: if self.tp_size > 1:
...@@ -125,23 +134,24 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): ...@@ -125,23 +134,24 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
if lora_bias is not None: if lora_bias is not None:
lora_bias = self.slice_bias(lora_bias) lora_bias = self.slice_bias(lora_bias)
self.lora_a_stacked[0][index, self.lora_a_stacked[0][index, 0, : lora_a.shape[0], : lora_a.shape[1]].copy_(
0, :lora_a.shape[0], :lora_a.shape[1]].copy_( lora_a, non_blocking=True
lora_a, non_blocking=True) )
self.lora_b_stacked[0][index, self.lora_b_stacked[0][index, 0, : lora_b.shape[0], : lora_b.shape[1]].copy_(
0, :lora_b.shape[0], :lora_b.shape[1]].copy_( lora_b, non_blocking=True
lora_b, non_blocking=True) )
if lora_bias is not None: if lora_bias is not None:
self.lora_bias_stacked = cast(
self.lora_bias_stacked = cast(tuple[torch.Tensor, ...], tuple[torch.Tensor, ...], self.lora_bias_stacked
self.lora_bias_stacked) )
assert len(self.lora_bias_stacked) assert len(self.lora_bias_stacked)
self.lora_bias_stacked[0][index, 0, :lora_bias.shape[0]].copy_( self.lora_bias_stacked[0][index, 0, : lora_bias.shape[0]].copy_(
lora_bias, non_blocking=True) lora_bias, non_blocking=True
)
def apply(self, def apply(
x: torch.Tensor, self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
bias: Optional[torch.Tensor] = None) -> torch.Tensor: ) -> torch.Tensor:
output = self.base_layer.quant_method.apply(self.base_layer, x, bias) output = self.base_layer.quant_method.apply(self.base_layer, x, bias)
# In transformers backend, x and output have extra batch dimension like # In transformers backend, x and output have extra batch dimension like
...@@ -151,10 +161,15 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): ...@@ -151,10 +161,15 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
output = output.flatten(0, 1) output = output.flatten(0, 1)
x = x.flatten(0, 1) x = x.flatten(0, 1)
lora_output: Optional[ lora_output: Optional[torch.Tensor] = self.punica_wrapper.add_lora_linear(
torch.Tensor] = self.punica_wrapper.add_lora_linear( output,
output, x, self.lora_a_stacked, self.lora_b_stacked, x,
self.lora_bias_stacked, 1.0, self.output_slices) self.lora_a_stacked,
self.lora_b_stacked,
self.lora_bias_stacked,
1.0,
self.output_slices,
)
if not current_platform.can_update_inplace(): if not current_platform.can_update_inplace():
output = lora_output output = lora_output
...@@ -162,7 +177,6 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): ...@@ -162,7 +177,6 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
@property @property
def weight(self) -> torch.Tensor: def weight(self) -> torch.Tensor:
# unquantizedLinear # unquantizedLinear
if hasattr(self.base_layer, "weight"): if hasattr(self.base_layer, "weight"):
return self.base_layer.weight return self.base_layer.weight
......
...@@ -12,8 +12,6 @@ from vllm.distributed import ( ...@@ -12,8 +12,6 @@ from vllm.distributed import (
split_tensor_along_last_dim, split_tensor_along_last_dim,
tensor_model_parallel_all_reduce, tensor_model_parallel_all_reduce,
) )
# yapf: disable
from vllm.model_executor.layers.linear import RowParallelLinear from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.platforms import current_platform from vllm.platforms import current_platform
...@@ -22,7 +20,6 @@ from .utils import _fully_sharded_can_replace, _not_fully_sharded_can_replace ...@@ -22,7 +20,6 @@ from .utils import _fully_sharded_can_replace, _not_fully_sharded_can_replace
class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA): class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
def __init__(self, base_layer: RowParallelLinear) -> None: def __init__(self, base_layer: RowParallelLinear) -> None:
super().__init__(base_layer) super().__init__(base_layer)
...@@ -33,11 +30,10 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA): ...@@ -33,11 +30,10 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
self.n_slices = 1 self.n_slices = 1
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor: def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
shard_size = self.input_size shard_size = self.input_size
start_idx = self.tp_rank * shard_size start_idx = self.tp_rank * shard_size
end_idx = (self.tp_rank + 1) * shard_size end_idx = (self.tp_rank + 1) * shard_size
lora_a = lora_a[:,start_idx:end_idx] lora_a = lora_a[:, start_idx:end_idx]
return lora_a return lora_a
def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor: def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
...@@ -66,7 +62,8 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA): ...@@ -66,7 +62,8 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
else: else:
# TODO: simplify code below # TODO: simplify code below
splitted_input = split_tensor_along_last_dim( splitted_input = split_tensor_along_last_dim(
input_, num_partitions=self.tp_size) input_, num_partitions=self.tp_size
)
input_parallel = splitted_input[self.tp_rank].contiguous() input_parallel = splitted_input[self.tp_rank].contiguous()
# Matrix multiply. # Matrix multiply.
...@@ -77,8 +74,11 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA): ...@@ -77,8 +74,11 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
output_ = output_parallel output_ = output_parallel
if not self.base_layer.skip_bias_add: if not self.base_layer.skip_bias_add:
output = (output_ + self.base_layer.bias output = (
if self.base_layer.bias is not None else output_) output_ + self.base_layer.bias
if self.base_layer.bias is not None
else output_
)
output_bias = None output_bias = None
else: else:
output = output_ output = output_
...@@ -101,11 +101,11 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA): ...@@ -101,11 +101,11 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
return type(source_layer) is RowParallelLinear return type(source_layer) is RowParallelLinear
# The following layer is based on the tensor parallelism strategy given in # The following layer is based on the tensor parallelism strategy given in
# Y. Sheng et al., S-LoRA: Serving Thousands of Concurrent LoRA Adapters. 2023, # Y. Sheng et al., S-LoRA: Serving Thousands of Concurrent LoRA Adapters. 2023,
# https://arxiv.org/abs/2311.03285. # https://arxiv.org/abs/2311.03285.
class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA): class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
""" """
Differs from RowParallelLinearWithLoRA by slicing the Differs from RowParallelLinearWithLoRA by slicing the
...@@ -120,28 +120,26 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA): ...@@ -120,28 +120,26 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
shard_size = self.lora_b_stacked[0].shape[2] shard_size = self.lora_b_stacked[0].shape[2]
start_idx = self.tp_rank * shard_size start_idx = self.tp_rank * shard_size
end_idx = (self.tp_rank + 1) * shard_size end_idx = (self.tp_rank + 1) * shard_size
lora_b = lora_b[ start_idx:end_idx,:] lora_b = lora_b[start_idx:end_idx, :]
return lora_b return lora_b
def slice_bias(self, bias: torch.Tensor) -> torch.Tensor: def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
if bias is None: if bias is None:
return bias return bias
self.lora_bias_stacked = cast(tuple[torch.Tensor, ...], self.lora_bias_stacked = cast(tuple[torch.Tensor, ...], self.lora_bias_stacked)
self.lora_bias_stacked)
shard_size = self.lora_bias_stacked[0].shape[2] shard_size = self.lora_bias_stacked[0].shape[2]
start_idx = self.tp_rank * shard_size start_idx = self.tp_rank * shard_size
end_idx = (self.tp_rank + 1) * shard_size end_idx = (self.tp_rank + 1) * shard_size
bias = bias[start_idx:end_idx] bias = bias[start_idx:end_idx]
return bias return bias
def apply(self, def apply(
x: torch.Tensor, self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
bias: Optional[torch.Tensor] = None) -> torch.Tensor: ) -> torch.Tensor:
output = self.base_layer.quant_method.apply(self.base_layer, x) output = self.base_layer.quant_method.apply(self.base_layer, x)
x = x.view(-1, x.shape[-1]) x = x.view(-1, x.shape[-1])
output, out_orig_shape = output.view(-1, output, out_orig_shape = output.view(-1, output.shape[-1]), output.shape
output.shape[-1]), output.shape
buffer = torch.zeros( buffer = torch.zeros(
(self.n_slices, x.shape[0], self.lora_a_stacked[0].shape[2]), (self.n_slices, x.shape[0], self.lora_a_stacked[0].shape[2]),
dtype=torch.float32, dtype=torch.float32,
...@@ -149,10 +147,11 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA): ...@@ -149,10 +147,11 @@ class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
) )
shrunk_buffer: Optional[torch.Tensor] = self.punica_wrapper.add_shrink( shrunk_buffer: Optional[torch.Tensor] = self.punica_wrapper.add_shrink(
buffer, x, self.lora_a_stacked, 1.0) buffer, x, self.lora_a_stacked, 1.0
)
if not current_platform.can_update_inplace(): if not current_platform.can_update_inplace():
buffer = shrunk_buffer buffer = shrunk_buffer
if self.tp_size>1: if self.tp_size > 1:
buffer = tensor_model_parallel_all_reduce(buffer) buffer = tensor_model_parallel_all_reduce(buffer)
# following S-LoRA, allows the fusing of all_gather and all_reduce # following S-LoRA, allows the fusing of all_gather and all_reduce
......
...@@ -19,8 +19,6 @@ from vllm.config.lora import LoRAConfig ...@@ -19,8 +19,6 @@ from vllm.config.lora import LoRAConfig
from vllm.logger import init_logger from vllm.logger import init_logger
# being imported for _all_lora_classes below # being imported for _all_lora_classes below
# yapf conflicts with isort for this block
# yapf: disable
from vllm.lora.layers import ( from vllm.lora.layers import (
BaseLayerWithLoRA, BaseLayerWithLoRA,
ColumnParallelLinearWithLoRA, ColumnParallelLinearWithLoRA,
...@@ -39,8 +37,6 @@ from vllm.lora.layers import ( ...@@ -39,8 +37,6 @@ from vllm.lora.layers import (
) )
from vllm.model_executor.layers.linear import LinearBase from vllm.model_executor.layers.linear import LinearBase
# yapf: enable
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
......
...@@ -14,8 +14,6 @@ import vllm.envs as envs ...@@ -14,8 +14,6 @@ import vllm.envs as envs
import vllm.model_executor.layers.fused_moe.modular_kernel as mk import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import _custom_ops as ops from vllm import _custom_ops as ops
from vllm.logger import init_logger from vllm.logger import init_logger
# yapf: disable
from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG, FUSED_MOE_UNQUANTIZED_CONFIG,
FusedMoEQuantConfig, FusedMoEQuantConfig,
...@@ -25,8 +23,6 @@ from vllm.model_executor.layers.fused_moe.cutlass_moe import ( ...@@ -25,8 +23,6 @@ from vllm.model_executor.layers.fused_moe.cutlass_moe import (
_valid_cutlass_block_scaled_grouped_gemm, _valid_cutlass_block_scaled_grouped_gemm,
run_cutlass_block_scaled_fused_experts, run_cutlass_block_scaled_fused_experts,
) )
# yapf: enable
from vllm.model_executor.layers.fused_moe.deep_gemm_moe import ( from vllm.model_executor.layers.fused_moe.deep_gemm_moe import (
_valid_deep_gemm, _valid_deep_gemm,
deep_gemm_moe_fp8, deep_gemm_moe_fp8,
......
...@@ -24,8 +24,6 @@ from vllm.distributed.eplb.eplb_state import EplbState ...@@ -24,8 +24,6 @@ from vllm.distributed.eplb.eplb_state import EplbState
from vllm.forward_context import ForwardContext, get_forward_context from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp from vllm.model_executor.custom_op import CustomOp
# yapf: disable
from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.config import (
FUSED_MOE_UNQUANTIZED_CONFIG, FUSED_MOE_UNQUANTIZED_CONFIG,
FusedMoEConfig, FusedMoEConfig,
...@@ -34,8 +32,6 @@ from vllm.model_executor.layers.fused_moe.config import ( ...@@ -34,8 +32,6 @@ from vllm.model_executor.layers.fused_moe.config import (
biased_moe_quant_config, biased_moe_quant_config,
) )
from vllm.model_executor.layers.fused_moe.fused_moe import zero_experts_compute_triton from vllm.model_executor.layers.fused_moe.fused_moe import zero_experts_compute_triton
# yapf: enable
from vllm.model_executor.layers.fused_moe.modular_kernel import ( from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEActivationFormat, FusedMoEActivationFormat,
FusedMoEModularKernel, FusedMoEModularKernel,
......
...@@ -10,7 +10,7 @@ import torch ...@@ -10,7 +10,7 @@ import torch
import vllm.envs as envs import vllm.envs as envs
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.utils import ( # yapf: disable from vllm.model_executor.layers.fused_moe.utils import (
_resize_cache, _resize_cache,
count_expert_num_tokens, count_expert_num_tokens,
) )
......
...@@ -24,8 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import ( ...@@ -24,8 +24,6 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase, QuantizeMethodBase,
) )
from vllm.model_executor.layers.utils import dispatch_unquantized_gemm from vllm.model_executor.layers.utils import dispatch_unquantized_gemm
# yapf: disable
from vllm.model_executor.parameter import ( from vllm.model_executor.parameter import (
BasevLLMParameter, BasevLLMParameter,
BlockQuantScaleParameter, BlockQuantScaleParameter,
...@@ -35,8 +33,6 @@ from vllm.model_executor.parameter import ( ...@@ -35,8 +33,6 @@ from vllm.model_executor.parameter import (
PerTensorScaleParameter, PerTensorScaleParameter,
RowvLLMParameter, RowvLLMParameter,
) )
# yapf: enable
from vllm.model_executor.utils import set_weight_attrs from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils import GiB_bytes from vllm.utils import GiB_bytes
......
...@@ -17,17 +17,12 @@ from vllm.model_executor.layers.quantization.kernels.mixed_precision import ( ...@@ -17,17 +17,12 @@ from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
from vllm.model_executor.layers.quantization.utils.marlin_utils import ( from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_repeat_scales_on_all_ranks, marlin_repeat_scales_on_all_ranks,
) )
# yapf conflicts with isort for this block
# yapf: disable
from vllm.model_executor.parameter import ( from vllm.model_executor.parameter import (
BasevLLMParameter, BasevLLMParameter,
ChannelQuantScaleParameter, ChannelQuantScaleParameter,
GroupQuantScaleParameter, GroupQuantScaleParameter,
PackedvLLMParameter, PackedvLLMParameter,
) )
# yapf: enable
from vllm.scalar_type import scalar_types from vllm.scalar_type import scalar_types
logger = init_logger(__name__) logger = init_logger(__name__)
......
...@@ -17,9 +17,6 @@ from vllm.model_executor.layers.quantization.kernels.mixed_precision import ( ...@@ -17,9 +17,6 @@ from vllm.model_executor.layers.quantization.kernels.mixed_precision import (
from vllm.model_executor.layers.quantization.utils.marlin_utils import ( from vllm.model_executor.layers.quantization.utils.marlin_utils import (
marlin_repeat_scales_on_all_ranks, marlin_repeat_scales_on_all_ranks,
) )
# yapf conflicts with isort for this block
# yapf: disable
from vllm.model_executor.parameter import ( from vllm.model_executor.parameter import (
BasevLLMParameter, BasevLLMParameter,
ChannelQuantScaleParameter, ChannelQuantScaleParameter,
...@@ -28,8 +25,6 @@ from vllm.model_executor.parameter import ( ...@@ -28,8 +25,6 @@ from vllm.model_executor.parameter import (
PackedvLLMParameter, PackedvLLMParameter,
RowvLLMParameter, RowvLLMParameter,
) )
# yapf: enable
from vllm.scalar_type import scalar_types from vllm.scalar_type import scalar_types
logger = init_logger(__name__) logger = init_logger(__name__)
......
...@@ -22,8 +22,6 @@ from vllm.distributed import ( ...@@ -22,8 +22,6 @@ from vllm.distributed import (
get_tensor_model_parallel_rank, get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size, get_tensor_model_parallel_world_size,
) )
# yapf: enable
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.linear import (
...@@ -51,8 +49,6 @@ from vllm.model_executor.utils import ( ...@@ -51,8 +49,6 @@ from vllm.model_executor.utils import (
) )
from vllm.platforms import current_platform from vllm.platforms import current_platform
# yapf conflicts with isort for this block
logger = init_logger(__name__) logger = init_logger(__name__)
......
...@@ -39,13 +39,10 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder ...@@ -39,13 +39,10 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.tensor_schema import TensorSchema, TensorShape
# yapf: disable
from .idefics2_vision_model import Idefics2VisionConfig from .idefics2_vision_model import Idefics2VisionConfig
from .idefics2_vision_model import ( from .idefics2_vision_model import (
Idefics2VisionTransformer as Idefics3VisionTransformer, Idefics2VisionTransformer as Idefics3VisionTransformer,
) )
# yapf: enable
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsQuant
from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel from .llama import LlamaDecoderLayer, LlamaMLP, LlamaModel
from .utils import ( from .utils import (
......
...@@ -22,8 +22,6 @@ from vllm.multimodal.inputs import ( ...@@ -22,8 +22,6 @@ from vllm.multimodal.inputs import (
MultiModalKwargsItems, MultiModalKwargsItems,
) )
from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems from vllm.multimodal.parse import ImageProcessorItems, ImageSize, MultiModalDataItems
# yapf: disable
from vllm.multimodal.processing import ( from vllm.multimodal.processing import (
BaseMultiModalProcessor, BaseMultiModalProcessor,
BaseProcessingInfo, BaseProcessingInfo,
...@@ -35,8 +33,6 @@ from vllm.multimodal.processing import ( ...@@ -35,8 +33,6 @@ from vllm.multimodal.processing import (
PromptUpdateDetails, PromptUpdateDetails,
replace_token_matches, replace_token_matches,
) )
# yapf: enable
from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.multimodal.profiling import BaseDummyInputsBuilder
from vllm.sequence import IntermediateTensors from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.tensor_schema import TensorSchema, TensorShape
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment