cpu_adam.py 1.36 KB
Newer Older
aiss's avatar
aiss committed
1
2
3
4
5
# Copyright (c) Microsoft Corporation.
# SPDX-License-Identifier: Apache-2.0

# DeepSpeed Team

6
import os
aiss's avatar
aiss committed
7
from .builder import TorchCPUOpBuilder
8
9


aiss's avatar
aiss committed
10
class CPUAdamBuilder(TorchCPUOpBuilder):
11
12
    BUILD_VAR = "DS_BUILD_CPU_ADAM"
    NAME = "cpu_adam"
aiss's avatar
aiss committed
13

14
15
16
17
18
19
20
    def __init__(self):
        super().__init__(name=self.NAME)

    def absolute_name(self):
        return f'deepspeed.ops.adam.{self.NAME}_op'

    def sources(self):
aiss's avatar
aiss committed
21
22
        if self.build_for_cpu:
            return ['csrc/adam/cpu_adam.cpp']
aiss's avatar
aiss committed
23

aiss's avatar
aiss committed
24
25
26
27
        return ['csrc/adam/cpu_adam.cpp', 'csrc/common/custom_cuda_kernel.cu']

    def libraries_args(self):
        args = super().libraries_args()
aiss's avatar
aiss committed
28
29
30
31
32
        if self.build_for_cpu:
            return args

        if not self.is_rocm_pytorch():
            args += ['curand']
aiss's avatar
aiss committed
33

aiss's avatar
aiss committed
34
        return args
35
36

    def include_paths(self):
aiss's avatar
aiss committed
37
        import torch
aiss's avatar
aiss committed
38
39
40
        if self.build_for_cpu:
            CUDA_INCLUDE = []
        elif not self.is_rocm_pytorch():
aiss's avatar
aiss committed
41
42
43
            CUDA_INCLUDE = [os.path.join(torch.utils.cpp_extension.CUDA_HOME, "include")]
        else:
            CUDA_INCLUDE = [
aiss's avatar
aiss committed
44
45
46
                os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include"),
                os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include", "rocrand"),
                os.path.join(torch.utils.cpp_extension.ROCM_HOME, "include", "hiprand"),
aiss's avatar
aiss committed
47
48
            ]
        return ['csrc/includes'] + CUDA_INCLUDE