Commit 62fe9a48 authored by zhuwenwen's avatar zhuwenwen
Browse files

use apply_rotary_emb_torch on z100l&k100 (qwen2-vl)

parent 48d8707e
...@@ -85,6 +85,7 @@ import re ...@@ -85,6 +85,7 @@ import re
from vllm import _custom_ops as ops from vllm import _custom_ops as ops
from vllm.model_executor.utils import pad_weight, gemm_bank_conf from vllm.model_executor.utils import pad_weight, gemm_bank_conf
from vllm.platforms import current_platform from vllm.platforms import current_platform
from vllm.utils import SUPPORT_TC
logger = init_logger(__name__) logger = init_logger(__name__)
...@@ -246,7 +247,7 @@ def apply_rotary_pos_emb_vision(t: torch.Tensor, ...@@ -246,7 +247,7 @@ def apply_rotary_pos_emb_vision(t: torch.Tensor,
apply_rotary_emb = apply_rotary_emb_torch apply_rotary_emb = apply_rotary_emb_torch
if current_platform.is_cuda(): if current_platform.is_cuda():
from vllm.vllm_flash_attn.layers.rotary import apply_rotary_emb from vllm.vllm_flash_attn.layers.rotary import apply_rotary_emb
if current_platform.is_rocm(): if current_platform.is_rocm() and SUPPORT_TC:
from flash_attn.layers.rotary import apply_rotary_emb from flash_attn.layers.rotary import apply_rotary_emb
output = apply_rotary_emb(t_, cos, sin).type_as(t) output = apply_rotary_emb(t_, cos, sin).type_as(t)
return output return output
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment