setup.py 18.2 KB
Newer Older
1
import torch
2
from setuptools import setup, find_packages
mcarilli's avatar
mcarilli committed
3
import subprocess
4

jjsjann123's avatar
jjsjann123 committed
5
import sys
Marek Kolodziej's avatar
Marek Kolodziej committed
6
import warnings
mcarilli's avatar
mcarilli committed
7
import os
jjsjann123's avatar
jjsjann123 committed
8

9
10
11
# ninja build does not work unless include_dirs are abs path
this_dir = os.path.dirname(os.path.abspath(__file__))

12
if not torch.cuda.is_available():
mcarilli's avatar
mcarilli committed
13
14
15
16
17
18
19
20
21
22
23
    # https://github.com/NVIDIA/apex/issues/486
    # Extension builds after https://github.com/pytorch/pytorch/pull/23408 attempt to query torch.cuda.get_device_capability(),
    # which will fail if you are compiling in an environment without visible GPUs (e.g. during an nvidia-docker build command).
    print('\nWarning: Torch did not find available GPUs on this system.\n',
          'If your intention is to cross-compile, this is not an error.\n'
          'By default, Apex will cross-compile for Pascal (compute capabilities 6.0, 6.1, 6.2),\n'
          'Volta (compute capability 7.0), and Turing (compute capability 7.5).\n'
          'If you wish to cross-compile for a single specific architecture,\n'
          'export TORCH_CUDA_ARCH_LIST="compute capability" before running setup.py.\n')
    if os.environ.get("TORCH_CUDA_ARCH_LIST", None) is None:
        os.environ["TORCH_CUDA_ARCH_LIST"] = "6.0;6.1;6.2;7.0;7.5"
24

25
26
27
28
29
print("torch.__version__  = ", torch.__version__)
TORCH_MAJOR = int(torch.__version__.split('.')[0])
TORCH_MINOR = int(torch.__version__.split('.')[1])

if TORCH_MAJOR == 0 and TORCH_MINOR < 4:
Michael Carilli's avatar
Michael Carilli committed
30
      raise RuntimeError("Apex requires Pytorch 0.4 or newer.\n" +
31
32
                         "The latest stable release can be obtained from https://pytorch.org/")

jjsjann123's avatar
jjsjann123 committed
33
34
35
cmdclass = {}
ext_modules = []

ptrblck's avatar
ptrblck committed
36
extras = {}
Marek Kolodziej's avatar
Marek Kolodziej committed
37
38
39
if "--pyprof" in sys.argv:
    with open('requirements.txt') as f:
        required_packages = f.read().splitlines()
ptrblck's avatar
ptrblck committed
40
        extras['pyprof'] = required_packages
Marek Kolodziej's avatar
Marek Kolodziej committed
41
42
43
44
45
46
47
    try:
        sys.argv.remove("--pyprof")
    except:
        pass
else:
    warnings.warn("Option --pyprof not specified. Not installing PyProf dependencies!")

48
if "--cpp_ext" in sys.argv or "--cuda_ext" in sys.argv:
Michael Carilli's avatar
Michael Carilli committed
49
50
    if TORCH_MAJOR == 0:
        raise RuntimeError("--cpp_ext requires Pytorch 1.0 or later, "
51
                           "found torch.__version__ = {}".format(torch.__version__))
52
53
54
55
56
57
58
59
60
61
    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

if "--cpp_ext" in sys.argv:
    from torch.utils.cpp_extension import CppExtension
    sys.argv.remove("--cpp_ext")
    ext_modules.append(
        CppExtension('apex_C',
                     ['csrc/flatten_unflatten.cpp',]))

mcarilli's avatar
mcarilli committed
62
63
64
65
66
67
68
69
70
71
72
73
74
75
def check_cuda_torch_binary_vs_bare_metal(cuda_dir):
    raw_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"], universal_newlines=True)
    output = raw_output.split()
    release_idx = output.index("release") + 1
    release = output[release_idx].split(".")
    bare_metal_major = release[0]
    bare_metal_minor = release[1][0]
    torch_binary_major = torch.version.cuda.split(".")[0]
    torch_binary_minor = torch.version.cuda.split(".")[1]

    print("\nCompiling cuda extensions with")
    print(raw_output + "from " + cuda_dir + "/bin\n")

    if (bare_metal_major != torch_binary_major) or (bare_metal_minor != torch_binary_minor):
Michael Carilli's avatar
Michael Carilli committed
76
77
78
79
80
81
        raise RuntimeError("Cuda extensions are being compiled with a version of Cuda that does " +
                           "not match the version used to compile Pytorch binaries.  " +
                           "Pytorch binaries were compiled with Cuda {}.\n".format(torch.version.cuda) +
                           "In some cases, a minor-version mismatch will not cause later errors:  " +
                           "https://github.com/NVIDIA/apex/pull/323#discussion_r287021798.  "
                           "You can try commenting out this check (at your own risk).")
mcarilli's avatar
mcarilli committed
82

mcarilli's avatar
mcarilli committed
83
84
85
86
87
88
89
90
91
92
93
# Set up macros for forward/backward compatibility hack around
# https://github.com/pytorch/pytorch/commit/4404762d7dd955383acee92e6f06b48144a0742e
# and
# https://github.com/NVIDIA/apex/issues/456
# https://github.com/pytorch/pytorch/commit/eb7b39e02f7d75c26d8a795ea8c7fd911334da7e#diff-4632522f237f1e4e728cb824300403ac
version_ge_1_1 = []
if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 0):
    version_ge_1_1 = ['-DVERSION_GE_1_1']
version_ge_1_3 = []
if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 2):
    version_ge_1_3 = ['-DVERSION_GE_1_3']
94
95
96
97
version_ge_1_5 = []
if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 4):
    version_ge_1_5 = ['-DVERSION_GE_1_5']
version_dependent_macros = version_ge_1_1 + version_ge_1_3 + version_ge_1_5
mcarilli's avatar
mcarilli committed
98

jjsjann123's avatar
jjsjann123 committed
99
if "--cuda_ext" in sys.argv:
100
    from torch.utils.cpp_extension import CUDAExtension
jjsjann123's avatar
jjsjann123 committed
101
    sys.argv.remove("--cuda_ext")
102
103

    if torch.utils.cpp_extension.CUDA_HOME is None:
Michael Carilli's avatar
Michael Carilli committed
104
        raise RuntimeError("--cuda_ext was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
105
    else:
mcarilli's avatar
mcarilli committed
106
107
        check_cuda_torch_binary_vs_bare_metal(torch.utils.cpp_extension.CUDA_HOME)

108
109
        ext_modules.append(
            CUDAExtension(name='amp_C',
110
                          sources=['csrc/amp_C_frontend.cpp',
Michael Carilli's avatar
Michael Carilli committed
111
                                   'csrc/multi_tensor_sgd_kernel.cu',
112
                                   'csrc/multi_tensor_scale_kernel.cu',
113
                                   'csrc/multi_tensor_axpby_kernel.cu',
114
                                   'csrc/multi_tensor_l2norm_kernel.cu',
115
116
                                   'csrc/multi_tensor_lamb_stage_1.cu',
                                   'csrc/multi_tensor_lamb_stage_2.cu',
117
                                   'csrc/multi_tensor_adam.cu',
Andrew Tulloch's avatar
Andrew Tulloch committed
118
                                   'csrc/multi_tensor_adagrad.cu',
119
120
                                   'csrc/multi_tensor_novograd.cu',
                                   'csrc/multi_tensor_lamb.cu'],
mcarilli's avatar
mcarilli committed
121
                          extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
Michael Carilli's avatar
Michael Carilli committed
122
                                              'nvcc':['-lineinfo',
123
                                                      '-O3',
124
                                                      # '--resource-usage',
mcarilli's avatar
mcarilli committed
125
                                                      '--use_fast_math'] + version_dependent_macros}))
126
127
128
        ext_modules.append(
            CUDAExtension(name='syncbn',
                          sources=['csrc/syncbn.cpp',
mcarilli's avatar
mcarilli committed
129
130
131
132
                                   'csrc/welford.cu'],
                          extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
                                              'nvcc':['-O3'] + version_dependent_macros}))

133
134
        ext_modules.append(
            CUDAExtension(name='fused_layer_norm_cuda',
135
136
                          sources=['csrc/layer_norm_cuda.cpp',
                                   'csrc/layer_norm_cuda_kernel.cu'],
mcarilli's avatar
mcarilli committed
137
                          extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
138
                                              'nvcc':['-maxrregcount=50',
mcarilli's avatar
mcarilli committed
139
                                                      '-O3',
mcarilli's avatar
mcarilli committed
140
                                                      '--use_fast_math'] + version_dependent_macros}))
jjsjann123's avatar
jjsjann123 committed
141

142
143
144
145
146
147
148
        ext_modules.append(
            CUDAExtension(name='mlp_cuda',
                          sources=['csrc/mlp.cpp',
                                   'csrc/mlp_cuda.cu'],
                          extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
                                              'nvcc':['-O3'] + version_dependent_macros}))

jjsjann123's avatar
jjsjann123 committed
149
150
151
152
153
154
155
156
if "--bnp" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--bnp")

    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

    if torch.utils.cpp_extension.CUDA_HOME is None:
157
        raise RuntimeError("--bnp was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
jjsjann123's avatar
jjsjann123 committed
158
159
160
161
162
163
164
    else:
        ext_modules.append(
            CUDAExtension(name='bnp',
                          sources=['apex/contrib/csrc/groupbn/batch_norm.cu',
                                   'apex/contrib/csrc/groupbn/ipc.cu',
                                   'apex/contrib/csrc/groupbn/interface.cpp',
                                   'apex/contrib/csrc/groupbn/batch_norm_add_relu.cu'],
165
                          include_dirs=[os.path.join(this_dir, 'csrc')],
mcarilli's avatar
mcarilli committed
166
                          extra_compile_args={'cxx': [] + version_dependent_macros,
jjsjann123's avatar
jjsjann123 committed
167
168
169
                                              'nvcc':['-DCUDA_HAS_FP16=1',
                                                      '-D__CUDA_NO_HALF_OPERATORS__',
                                                      '-D__CUDA_NO_HALF_CONVERSIONS__',
170
                                                      '-D__CUDA_NO_HALF2_OPERATORS__'] + version_dependent_macros}))
jjsjann123's avatar
jjsjann123 committed
171

172
173
174
175
176
177
178
179
180
181
182
183
184
185
if "--xentropy" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--xentropy")

    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

    if torch.utils.cpp_extension.CUDA_HOME is None:
        raise RuntimeError("--xentropy was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
    else:
        ext_modules.append(
            CUDAExtension(name='xentropy_cuda',
                          sources=['apex/contrib/csrc/xentropy/interface.cpp',
                                   'apex/contrib/csrc/xentropy/xentropy_kernel.cu'],
186
                          include_dirs=[os.path.join(this_dir, 'csrc')],
mcarilli's avatar
mcarilli committed
187
188
                          extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
                                              'nvcc':['-O3'] + version_dependent_macros}))
189

190
191
192
193
194
195
196
197
198
199
200
201
202
203
if "--deprecated_fused_adam" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--deprecated_fused_adam")

    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

    if torch.utils.cpp_extension.CUDA_HOME is None:
        raise RuntimeError("--deprecated_fused_adam was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
    else:
        ext_modules.append(
            CUDAExtension(name='fused_adam_cuda',
                          sources=['apex/contrib/csrc/optimizers/fused_adam_cuda.cpp',
                                   'apex/contrib/csrc/optimizers/fused_adam_cuda_kernel.cu'],
204
                          include_dirs=[os.path.join(this_dir, 'csrc')],
205
206
207
208
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros,
                                              'nvcc':['-O3',
                                                      '--use_fast_math'] + version_dependent_macros}))

209
210
211
212
213
214
215
216
217
218
219
if "--deprecated_fused_lamb" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--deprecated_fused_lamb")

    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

    if torch.utils.cpp_extension.CUDA_HOME is None:
        raise RuntimeError("--deprecated_fused_lamb was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
    else:
        ext_modules.append(
Kexin Yu's avatar
Kexin Yu committed
220
            CUDAExtension(name='fused_lamb_cuda',
221
                          sources=['apex/contrib/csrc/optimizers/fused_lamb_cuda.cpp',
Kexin Yu's avatar
Kexin Yu committed
222
223
                                   'apex/contrib/csrc/optimizers/fused_lamb_cuda_kernel.cu',
                                   'csrc/multi_tensor_l2norm_kernel.cu'],
224
225
226
227
228
                          include_dirs=[os.path.join(this_dir, 'csrc')],
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros,
                                              'nvcc':['-O3',
                                                      '--use_fast_math'] + version_dependent_macros}))

ptrblck's avatar
ptrblck committed
229
230
231
232
233
234
# Check, if ATen/CUDAGenerator.h is found, otherwise use the new ATen/CUDAGeneratorImpl.h, due to breaking change in https://github.com/pytorch/pytorch/pull/36026 
generator_flag = []
torch_dir = torch.__path__[0]
if os.path.exists(os.path.join(torch_dir, 'include', 'ATen', 'CUDAGenerator.h')):
    generator_flag = ['-DOLD_GENERATOR']

235
236
237
238
239
if "--fast_multihead_attn" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--fast_multihead_attn")

    from torch.utils.cpp_extension import BuildExtension
240
    cmdclass['build_ext'] = BuildExtension.with_options(use_ninja=False)
241
242
243
244
245
246
247

    if torch.utils.cpp_extension.CUDA_HOME is None:
        raise RuntimeError("--fast_multihead_attn was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
    else:
        subprocess.run(["git", "submodule", "update", "--init", "apex/contrib/csrc/multihead_attn/cutlass"])
        ext_modules.append(
            CUDAExtension(name='fast_self_multihead_attn',
248
                          sources=['apex/contrib/csrc/multihead_attn/self_multihead_attn.cpp',
249
                                   'apex/contrib/csrc/multihead_attn/self_multihead_attn_cuda.cu'],
ptrblck's avatar
ptrblck committed
250
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros + generator_flag,
251
                                              'nvcc':['-O3',
252
                                                      '-gencode', 'arch=compute_70,code=sm_70',
253
254
255
256
257
                                                      '-I./apex/contrib/csrc/multihead_attn/cutlass/',
                                                      '-U__CUDA_NO_HALF_OPERATORS__',
                                                      '-U__CUDA_NO_HALF_CONVERSIONS__',
                                                      '--expt-relaxed-constexpr',
                                                      '--expt-extended-lambda',
ptrblck's avatar
ptrblck committed
258
                                                      '--use_fast_math'] + version_dependent_macros + generator_flag}))
259
260
        ext_modules.append(
            CUDAExtension(name='fast_self_multihead_attn_norm_add',
261
                          sources=['apex/contrib/csrc/multihead_attn/self_multihead_attn_norm_add.cpp',
262
                                   'apex/contrib/csrc/multihead_attn/self_multihead_attn_norm_add_cuda.cu'],
ptrblck's avatar
ptrblck committed
263
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros + generator_flag,
264
                                              'nvcc':['-O3',
265
                                                      '-gencode', 'arch=compute_70,code=sm_70',
266
267
268
269
270
                                                      '-I./apex/contrib/csrc/multihead_attn/cutlass/',
                                                      '-U__CUDA_NO_HALF_OPERATORS__',
                                                      '-U__CUDA_NO_HALF_CONVERSIONS__',
                                                      '--expt-relaxed-constexpr',
                                                      '--expt-extended-lambda',
ptrblck's avatar
ptrblck committed
271
                                                      '--use_fast_math'] + version_dependent_macros + generator_flag}))
272
273
        ext_modules.append(
            CUDAExtension(name='fast_encdec_multihead_attn',
274
                          sources=['apex/contrib/csrc/multihead_attn/encdec_multihead_attn.cpp',
275
                                   'apex/contrib/csrc/multihead_attn/encdec_multihead_attn_cuda.cu'],
ptrblck's avatar
ptrblck committed
276
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros + generator_flag,
277
                                              'nvcc':['-O3',
278
                                                      '-gencode', 'arch=compute_70,code=sm_70',
279
280
281
282
283
                                                      '-I./apex/contrib/csrc/multihead_attn/cutlass/',
                                                      '-U__CUDA_NO_HALF_OPERATORS__',
                                                      '-U__CUDA_NO_HALF_CONVERSIONS__',
                                                      '--expt-relaxed-constexpr',
                                                      '--expt-extended-lambda',
ptrblck's avatar
ptrblck committed
284
                                                      '--use_fast_math'] + version_dependent_macros + generator_flag}))
285
286
        ext_modules.append(
            CUDAExtension(name='fast_encdec_multihead_attn_norm_add',
287
                          sources=['apex/contrib/csrc/multihead_attn/encdec_multihead_attn_norm_add.cpp',
288
                                   'apex/contrib/csrc/multihead_attn/encdec_multihead_attn_norm_add_cuda.cu'],
ptrblck's avatar
ptrblck committed
289
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros + generator_flag,
290
                                              'nvcc':['-O3',
291
                                                      '-gencode', 'arch=compute_70,code=sm_70',
292
293
294
295
296
                                                      '-I./apex/contrib/csrc/multihead_attn/cutlass/',
                                                      '-U__CUDA_NO_HALF_OPERATORS__',
                                                      '-U__CUDA_NO_HALF_CONVERSIONS__',
                                                      '--expt-relaxed-constexpr',
                                                      '--expt-extended-lambda',
ptrblck's avatar
ptrblck committed
297
                                                      '--use_fast_math'] + version_dependent_macros + generator_flag}))
298

Christian Sarofeen's avatar
Christian Sarofeen committed
299
setup(
300
301
    name='apex',
    version='0.1',
302
303
304
305
    packages=find_packages(exclude=('build',
                                    'csrc',
                                    'include',
                                    'tests',
306
307
308
309
310
                                    'dist',
                                    'docs',
                                    'tests',
                                    'examples',
                                    'apex.egg-info',)),
Christian Sarofeen's avatar
Christian Sarofeen committed
311
    description='PyTorch Extensions written by NVIDIA',
jjsjann123's avatar
jjsjann123 committed
312
313
    ext_modules=ext_modules,
    cmdclass=cmdclass,
ptrblck's avatar
ptrblck committed
314
    extras_require=extras,
Christian Sarofeen's avatar
Christian Sarofeen committed
315
)