Unverified Commit 4134312b authored by Lucas Wilkinson's avatar Lucas Wilkinson Committed by GitHub
Browse files

[BugFix] ChunkedLocalAttention is currently not CG compatible (#26034)


Signed-off-by: default avatarLucas Wilkinson <lwilkins@redhat.com>
parent da554f93
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools import functools
from typing import List, Optional from typing import ClassVar, List, Optional
import torch import torch
...@@ -12,8 +12,8 @@ from vllm.attention.selector import get_attn_backend ...@@ -12,8 +12,8 @@ from vllm.attention.selector import get_attn_backend
from vllm.config import CacheConfig from vllm.config import CacheConfig
from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.v1.attention.backends.utils import ( from vllm.v1.attention.backends.utils import (
CommonAttentionMetadata, make_local_attention_virtual_batches, AttentionCGSupport, CommonAttentionMetadata,
subclass_attention_backend) make_local_attention_virtual_batches, subclass_attention_backend)
from ..layer import Attention from ..layer import Attention
...@@ -29,6 +29,8 @@ def create_chunked_local_attention_backend( ...@@ -29,6 +29,8 @@ def create_chunked_local_attention_backend(
underlying_builder = underlying_attn_backend.get_builder_cls() underlying_builder = underlying_attn_backend.get_builder_cls()
class ChunkedLocalAttentionBuilder(underlying_builder): # type: ignore class ChunkedLocalAttentionBuilder(underlying_builder): # type: ignore
cudagraph_support: ClassVar[AttentionCGSupport] = \
AttentionCGSupport.NEVER
def build(self, def build(self,
common_prefix_len: int, common_prefix_len: int,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment