tpu.py 1.39 KB
Newer Older
1
import os
2
from typing import TYPE_CHECKING
3

4
5
import torch

6
7
8
from vllm.logger import init_logger

from .interface import Platform, PlatformEnum, _Backend
9

10
11
12
13
if TYPE_CHECKING:
    from vllm.config import VllmConfig
else:
    VllmConfig = None
14

15
16
logger = init_logger(__name__)

17
18
19
20

class TpuPlatform(Platform):
    _enum = PlatformEnum.TPU

21
22
23
24
25
26
    @classmethod
    def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
        if selected_backend != _Backend.PALLAS:
            logger.info("Cannot use %s backend on TPU.", selected_backend)
        return _Backend.PALLAS

27
28
29
30
    @classmethod
    def get_device_name(cls, device_id: int = 0) -> str:
        raise NotImplementedError

31
32
33
34
    @classmethod
    def get_device_total_memory(cls, device_id: int = 0) -> int:
        raise NotImplementedError

35
36
    @classmethod
    def inference_mode(cls):
37
        return torch.no_grad()
38
39
40
41
42
43
44
45
46

    @classmethod
    def check_and_update_config(cls, vllm_config: VllmConfig) -> None:
        from vllm.config import CompilationLevel
        compilation_config = vllm_config.compilation_config
        if "VLLM_TORCH_COMPILE_LEVEL" not in os.environ:
            compilation_config.level = CompilationLevel.DYNAMO_ONCE
        assert compilation_config.level < CompilationLevel.PIECEWISE,\
            "TPU does not support Inductor."
47
48
49

        if compilation_config.backend == "":
            compilation_config.backend = "openxla"