tpu.py 5.31 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
from typing import TYPE_CHECKING, Optional, Union
4

5
6
import torch

7
import vllm.envs as envs
8
from vllm.inputs import PromptType
9
10
11
from vllm.logger import init_logger

from .interface import Platform, PlatformEnum, _Backend
12

13
if TYPE_CHECKING:
14
    from vllm.config import ModelConfig, VllmConfig
15
16
17
    from vllm.lora.request import LoRARequest
    from vllm.pooling_params import PoolingParams
    from vllm.sampling_params import SamplingParams
18
else:
19
    ModelConfig = None
20
    VllmConfig = None
21
22
23
    LoRARequest = None
    PoolingParams = None
    SamplingParams = None
24

25
26
logger = init_logger(__name__)

27
28
29

class TpuPlatform(Platform):
    _enum = PlatformEnum.TPU
30
    device_name: str = "tpu"
31
    device_type: str = "tpu"
32
    dispatch_key: str = "XLA"
33
    ray_device_key: str = "TPU"
34
    device_control_env_var: str = "TPU_VISIBLE_CHIPS"
35

36
37
38
    supported_quantization: list[str] = [
        "tpu_int8", "compressed-tensors", "compressed_tensors"
    ]
39

40
41
42
43
    additional_env_vars: list[str] = [
        "TPU_CHIPS_PER_HOST_BOUNDS", "TPU_HOST_BOUNDS"
    ]

44
    @classmethod
45
46
    def get_attn_backend_cls(cls, selected_backend: _Backend, head_size: int,
                             dtype: torch.dtype, kv_cache_dtype: Optional[str],
47
48
                             block_size: int, use_v1: bool,
                             use_mla: bool) -> str:
49
50
        if (selected_backend != _Backend.PALLAS
                and selected_backend != _Backend.PALLAS_VLLM_V1):
51
            logger.info("Cannot use %s backend on TPU.", selected_backend)
52
53
54
55
56
57
58

        if use_v1:
            logger.info("Using Pallas V1 backend.")
            return "vllm.v1.attention.backends.pallas.PallasAttentionBackend"
        else:
            logger.info("Using Pallas backend.")
            return "vllm.attention.backends.pallas.PallasAttentionBackend"
59

60
61
    @classmethod
    def get_device_name(cls, device_id: int = 0) -> str:
62
        return "tpu"
63

64
65
66
67
    @classmethod
    def get_device_total_memory(cls, device_id: int = 0) -> int:
        raise NotImplementedError

68
69
    @classmethod
    def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
70
        return not envs.VLLM_USE_V1
71

72
73
    @classmethod
    def inference_mode(cls):
74
        return torch.no_grad()
75
76
77
78

    @classmethod
    def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
        from vllm.config import CompilationLevel
79
80
81
82
83

        cache_config = vllm_config.cache_config
        if cache_config and cache_config.block_size is None:
            cache_config.block_size = 16

84
        compilation_config = vllm_config.compilation_config
85
86
87
88

        # TPU only supports DYNAMO_ONCE compilation level
        if compilation_config.level != CompilationLevel.DYNAMO_ONCE:
            logger.info("[TPU] Forcing DYNAMO_ONCE compilation level")
89
            compilation_config.level = CompilationLevel.DYNAMO_ONCE
90
91
92

        if compilation_config.backend == "":
            compilation_config.backend = "openxla"
93
94
95
96

        assert vllm_config.speculative_config is None, \
            "TPU does not support speculative decoding"

97
98
99
100
101
102
        if vllm_config.model_config.dtype in (torch.float16, torch.float32):
            logger.warning(
                "The TPU backend currently does not support %s. "
                "Using bfloat16 instead.", vllm_config.model_config.dtype)
            vllm_config.model_config.dtype = torch.bfloat16

103
104
105
        parallel_config = vllm_config.parallel_config
        scheduler_config = vllm_config.scheduler_config
        if parallel_config.worker_cls == "auto":
106
107
108
109
110
111
112
            if scheduler_config.is_multi_step:
                if envs.VLLM_USE_V1:
                    raise NotImplementedError(
                        "Multi-step scheduling is not supported (and not "
                        "needed) on vLLM V1. Please launch without "
                        "--num-scheduler-steps.")
                else:
113
114
                    parallel_config.worker_cls = \
                        "vllm.worker.multi_step_tpu_worker.MultiStepTPUWorker"
115
116
117
118
            else:
                if envs.VLLM_USE_V1:
                    parallel_config.worker_cls = \
                        "vllm.v1.worker.tpu_worker.TPUWorker"
119
120
121
122
123
124
125
126
127
128
129
                else:
                    parallel_config.worker_cls = \
                        "vllm.worker.tpu_worker.TPUWorker"

        assert not vllm_config.speculative_config, (
            "Speculative decoding is not yet supported for TPU backend")

    @classmethod
    def is_pin_memory_available(cls):
        logger.warning("Pin memory is not supported on TPU.")
        return False
130
131
132
133

    @classmethod
    def get_device_communicator_cls(cls) -> str:
        return "vllm.distributed.device_communicators.tpu_communicator.TpuCommunicator"  # noqa
134
135
136
137

    @classmethod
    def use_all_gather(cls) -> bool:
        return True
138
139
140
141
142

    @classmethod
    def supports_v1(cls, model_config: ModelConfig) -> bool:
        # V1 support on TPU is experimental
        return True
143
144

    @classmethod
145
146
147
148
149
150
151
152
153
154
    def validate_request(
        cls,
        prompt: PromptType,
        params: Union[SamplingParams, PoolingParams],
    ) -> None:
        """Raises if this request is unsupported on this platform"""
        if isinstance(params,
                      SamplingParams) and params.guided_decoding is not None:
            raise ValueError("Structured output is not supported on "
                             f"{cls.device_name}.")