setup.py 21 KB
Newer Older
1
import torch
2
from setuptools import setup, find_packages
mcarilli's avatar
mcarilli committed
3
import subprocess
4

jjsjann123's avatar
jjsjann123 committed
5
import sys
Marek Kolodziej's avatar
Marek Kolodziej committed
6
import warnings
mcarilli's avatar
mcarilli committed
7
import os
jjsjann123's avatar
jjsjann123 committed
8

9
10
from torch.utils.hipify import hipify_python

11
12
13
# ninja build does not work unless include_dirs are abs path
this_dir = os.path.dirname(os.path.abspath(__file__))

14
if not torch.cuda.is_available():
mcarilli's avatar
mcarilli committed
15
16
17
18
19
20
21
22
23
24
25
    # https://github.com/NVIDIA/apex/issues/486
    # Extension builds after https://github.com/pytorch/pytorch/pull/23408 attempt to query torch.cuda.get_device_capability(),
    # which will fail if you are compiling in an environment without visible GPUs (e.g. during an nvidia-docker build command).
    print('\nWarning: Torch did not find available GPUs on this system.\n',
          'If your intention is to cross-compile, this is not an error.\n'
          'By default, Apex will cross-compile for Pascal (compute capabilities 6.0, 6.1, 6.2),\n'
          'Volta (compute capability 7.0), and Turing (compute capability 7.5).\n'
          'If you wish to cross-compile for a single specific architecture,\n'
          'export TORCH_CUDA_ARCH_LIST="compute capability" before running setup.py.\n')
    if os.environ.get("TORCH_CUDA_ARCH_LIST", None) is None:
        os.environ["TORCH_CUDA_ARCH_LIST"] = "6.0;6.1;6.2;7.0;7.5"
26

27
28
29
30
31
print("torch.__version__  = ", torch.__version__)
TORCH_MAJOR = int(torch.__version__.split('.')[0])
TORCH_MINOR = int(torch.__version__.split('.')[1])

if TORCH_MAJOR == 0 and TORCH_MINOR < 4:
Michael Carilli's avatar
Michael Carilli committed
32
      raise RuntimeError("Apex requires Pytorch 0.4 or newer.\n" +
33
34
                         "The latest stable release can be obtained from https://pytorch.org/")

jjsjann123's avatar
jjsjann123 committed
35
36
37
cmdclass = {}
ext_modules = []

ptrblck's avatar
ptrblck committed
38
extras = {}
Marek Kolodziej's avatar
Marek Kolodziej committed
39
40
41
if "--pyprof" in sys.argv:
    with open('requirements.txt') as f:
        required_packages = f.read().splitlines()
ptrblck's avatar
ptrblck committed
42
        extras['pyprof'] = required_packages
Marek Kolodziej's avatar
Marek Kolodziej committed
43
44
45
46
47
48
49
    try:
        sys.argv.remove("--pyprof")
    except:
        pass
else:
    warnings.warn("Option --pyprof not specified. Not installing PyProf dependencies!")

50
if "--cpp_ext" in sys.argv or "--cuda_ext" in sys.argv:
Michael Carilli's avatar
Michael Carilli committed
51
52
    if TORCH_MAJOR == 0:
        raise RuntimeError("--cpp_ext requires Pytorch 1.0 or later, "
53
                           "found torch.__version__ = {}".format(torch.__version__))
54
55
56
57
58
59
60
61
62
63
    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

if "--cpp_ext" in sys.argv:
    from torch.utils.cpp_extension import CppExtension
    sys.argv.remove("--cpp_ext")
    ext_modules.append(
        CppExtension('apex_C',
                     ['csrc/flatten_unflatten.cpp',]))

mcarilli's avatar
mcarilli committed
64
65
66
67
68
69
70
71
72
73
74
75
76
77
def check_cuda_torch_binary_vs_bare_metal(cuda_dir):
    raw_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"], universal_newlines=True)
    output = raw_output.split()
    release_idx = output.index("release") + 1
    release = output[release_idx].split(".")
    bare_metal_major = release[0]
    bare_metal_minor = release[1][0]
    torch_binary_major = torch.version.cuda.split(".")[0]
    torch_binary_minor = torch.version.cuda.split(".")[1]

    print("\nCompiling cuda extensions with")
    print(raw_output + "from " + cuda_dir + "/bin\n")

    if (bare_metal_major != torch_binary_major) or (bare_metal_minor != torch_binary_minor):
Michael Carilli's avatar
Michael Carilli committed
78
79
80
81
82
83
        raise RuntimeError("Cuda extensions are being compiled with a version of Cuda that does " +
                           "not match the version used to compile Pytorch binaries.  " +
                           "Pytorch binaries were compiled with Cuda {}.\n".format(torch.version.cuda) +
                           "In some cases, a minor-version mismatch will not cause later errors:  " +
                           "https://github.com/NVIDIA/apex/pull/323#discussion_r287021798.  "
                           "You can try commenting out this check (at your own risk).")
mcarilli's avatar
mcarilli committed
84

mcarilli's avatar
mcarilli committed
85
86
87
88
89
90
91
92
93
94
95
# Set up macros for forward/backward compatibility hack around
# https://github.com/pytorch/pytorch/commit/4404762d7dd955383acee92e6f06b48144a0742e
# and
# https://github.com/NVIDIA/apex/issues/456
# https://github.com/pytorch/pytorch/commit/eb7b39e02f7d75c26d8a795ea8c7fd911334da7e#diff-4632522f237f1e4e728cb824300403ac
version_ge_1_1 = []
if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 0):
    version_ge_1_1 = ['-DVERSION_GE_1_1']
version_ge_1_3 = []
if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 2):
    version_ge_1_3 = ['-DVERSION_GE_1_3']
96
97
98
99
version_ge_1_5 = []
if (TORCH_MAJOR > 1) or (TORCH_MAJOR == 1 and TORCH_MINOR > 4):
    version_ge_1_5 = ['-DVERSION_GE_1_5']
version_dependent_macros = version_ge_1_1 + version_ge_1_3 + version_ge_1_5
mcarilli's avatar
mcarilli committed
100

jjsjann123's avatar
jjsjann123 committed
101
if "--cuda_ext" in sys.argv:
102
    from torch.utils.cpp_extension import CUDAExtension
jjsjann123's avatar
jjsjann123 committed
103
    sys.argv.remove("--cuda_ext")
104
105
106
107
108
    
    is_rocm_pytorch = False
    if torch.__version__ >= '1.5':
        from torch.utils.cpp_extension import ROCM_HOME
        is_rocm_pytorch = True if ((torch.version.hip is not None) and (ROCM_HOME is not None)) else False
109

110
    if torch.utils.cpp_extension.CUDA_HOME is None and (not is_rocm_pytorch):
Michael Carilli's avatar
Michael Carilli committed
111
        raise RuntimeError("--cuda_ext was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
112
    else:
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
        if not is_rocm_pytorch:
            check_cuda_torch_binary_vs_bare_metal(torch.utils.cpp_extension.CUDA_HOME)

        if is_rocm_pytorch:
            import shutil
            with hipify_python.GeneratedFileCleaner(keep_intermediates=True) as clean_ctx:
                hipify_python.hipify(project_directory=this_dir, output_directory=this_dir, includes="csrc/*",
                                        show_detailed=True, is_pytorch_extension=True, clean_ctx=clean_ctx)
            shutil.copy("csrc/compat.h", "csrc/hip/compat.h")
            shutil.copy("csrc/type_shim.h", "csrc/hip/type_shim.h")

        if not is_rocm_pytorch:
            ext_modules.append(
                CUDAExtension(name='amp_C',
                              sources=['csrc/amp_C_frontend.cpp',
                                       'csrc/multi_tensor_sgd_kernel.cu',
                                       'csrc/multi_tensor_scale_kernel.cu',
                                       'csrc/multi_tensor_axpby_kernel.cu',
                                       'csrc/multi_tensor_l2norm_kernel.cu',
                                       'csrc/multi_tensor_lamb_stage_1.cu',
                                       'csrc/multi_tensor_lamb_stage_2.cu',
                                       'csrc/multi_tensor_adam.cu',
135
                                       'csrc/multi_tensor_adagrad.cu',
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
                                       'csrc/multi_tensor_novograd.cu',
                                       'csrc/multi_tensor_lamb.cu'],
                              extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
                                                  'nvcc':['-lineinfo',
                                                          '-O3',
                                                          # '--resource-usage',
                                                          '--use_fast_math'] + version_dependent_macros}))
        else:
            print ("INFO: Building Multitensor apply extension")
            ext_modules.append(
                CUDAExtension(name='amp_C',
                              sources=['csrc/amp_C_frontend.cpp',
                                       'csrc/hip/multi_tensor_sgd_kernel.hip',
                                       'csrc/hip/multi_tensor_scale_kernel.hip',
                                       'csrc/hip/multi_tensor_axpby_kernel.hip',
                                       'csrc/hip/multi_tensor_l2norm_kernel.hip',
                                       'csrc/hip/multi_tensor_lamb_stage_1.hip',
                                       'csrc/hip/multi_tensor_lamb_stage_2.hip',
                                       'csrc/hip/multi_tensor_adam.hip',
155
                                       'csrc/hip/multi_tensor_adagrad.hip',
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
                                       'csrc/hip/multi_tensor_novograd.hip',
                                       'csrc/hip/multi_tensor_lamb.hip'],
                              extra_compile_args={'cxx' : ['-O3'] + version_dependent_macros,
                                                  'nvcc': []}))

        if not is_rocm_pytorch:
            ext_modules.append(
                CUDAExtension(name='syncbn',
                              sources=['csrc/syncbn.cpp',
                                       'csrc/welford.cu'],
                              extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
                                                  'nvcc':['-O3'] + version_dependent_macros}))
        else:
            print ("INFO: Skipping syncbn extension.")

    
        if not is_rocm_pytorch:
            ext_modules.append(
                CUDAExtension(name='fused_layer_norm_cuda',
                              sources=['csrc/layer_norm_cuda.cpp',
                                       'csrc/layer_norm_cuda_kernel.cu'],
                              extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
                                                  'nvcc':['-maxrregcount=50',
                                                          '-O3',
                                                          '--use_fast_math'] + version_dependent_macros}))
        else:
182
183
184
185
186
187
188
            print ("INFO: Building FusedLayerNorm extension.")
            ext_modules.append(
                CUDAExtension(name='fused_layer_norm_cuda',
                              sources=['csrc/layer_norm_cuda.cpp',
                                       'csrc/hip/layer_norm_hip_kernel.hip'],
                              extra_compile_args={'cxx' : ['-O3'] + version_dependent_macros,
                                                  'nvcc' : []}))
189
190
191
192
193
194
195
196
197
198

        if not is_rocm_pytorch:
            ext_modules.append(
                CUDAExtension(name='mlp_cuda',
                              sources=['csrc/mlp.cpp',
                                       'csrc/mlp_cuda.cu'],
                              extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
                                                  'nvcc':['-O3'] + version_dependent_macros}))
        else:
            print ("INFO: Skipping MLP extension")
199

jjsjann123's avatar
jjsjann123 committed
200
201
202
203
204
205
206
207
if "--bnp" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--bnp")

    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

    if torch.utils.cpp_extension.CUDA_HOME is None:
208
        raise RuntimeError("--bnp was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
jjsjann123's avatar
jjsjann123 committed
209
210
211
212
213
214
215
    else:
        ext_modules.append(
            CUDAExtension(name='bnp',
                          sources=['apex/contrib/csrc/groupbn/batch_norm.cu',
                                   'apex/contrib/csrc/groupbn/ipc.cu',
                                   'apex/contrib/csrc/groupbn/interface.cpp',
                                   'apex/contrib/csrc/groupbn/batch_norm_add_relu.cu'],
216
                          include_dirs=[os.path.join(this_dir, 'csrc')],
mcarilli's avatar
mcarilli committed
217
                          extra_compile_args={'cxx': [] + version_dependent_macros,
jjsjann123's avatar
jjsjann123 committed
218
219
220
                                              'nvcc':['-DCUDA_HAS_FP16=1',
                                                      '-D__CUDA_NO_HALF_OPERATORS__',
                                                      '-D__CUDA_NO_HALF_CONVERSIONS__',
221
                                                      '-D__CUDA_NO_HALF2_OPERATORS__'] + version_dependent_macros}))
jjsjann123's avatar
jjsjann123 committed
222

223
224
225
226
227
228
229
230
231
232
233
234
235
236
if "--xentropy" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--xentropy")

    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

    if torch.utils.cpp_extension.CUDA_HOME is None:
        raise RuntimeError("--xentropy was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
    else:
        ext_modules.append(
            CUDAExtension(name='xentropy_cuda',
                          sources=['apex/contrib/csrc/xentropy/interface.cpp',
                                   'apex/contrib/csrc/xentropy/xentropy_kernel.cu'],
237
                          include_dirs=[os.path.join(this_dir, 'csrc')],
mcarilli's avatar
mcarilli committed
238
239
                          extra_compile_args={'cxx': ['-O3'] + version_dependent_macros,
                                              'nvcc':['-O3'] + version_dependent_macros}))
240

241
242
243
244
245
246
247
248
249
250
251
252
253
254
if "--deprecated_fused_adam" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--deprecated_fused_adam")

    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

    if torch.utils.cpp_extension.CUDA_HOME is None:
        raise RuntimeError("--deprecated_fused_adam was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
    else:
        ext_modules.append(
            CUDAExtension(name='fused_adam_cuda',
                          sources=['apex/contrib/csrc/optimizers/fused_adam_cuda.cpp',
                                   'apex/contrib/csrc/optimizers/fused_adam_cuda_kernel.cu'],
255
                          include_dirs=[os.path.join(this_dir, 'csrc')],
256
257
258
259
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros,
                                              'nvcc':['-O3',
                                                      '--use_fast_math'] + version_dependent_macros}))

260
261
262
263
264
265
266
267
268
269
270
if "--deprecated_fused_lamb" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--deprecated_fused_lamb")

    from torch.utils.cpp_extension import BuildExtension
    cmdclass['build_ext'] = BuildExtension

    if torch.utils.cpp_extension.CUDA_HOME is None:
        raise RuntimeError("--deprecated_fused_lamb was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
    else:
        ext_modules.append(
Kexin Yu's avatar
Kexin Yu committed
271
            CUDAExtension(name='fused_lamb_cuda',
272
                          sources=['apex/contrib/csrc/optimizers/fused_lamb_cuda.cpp',
Kexin Yu's avatar
Kexin Yu committed
273
274
                                   'apex/contrib/csrc/optimizers/fused_lamb_cuda_kernel.cu',
                                   'csrc/multi_tensor_l2norm_kernel.cu'],
275
276
277
278
279
                          include_dirs=[os.path.join(this_dir, 'csrc')],
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros,
                                              'nvcc':['-O3',
                                                      '--use_fast_math'] + version_dependent_macros}))

ptrblck's avatar
ptrblck committed
280
281
282
283
284
285
# Check, if ATen/CUDAGenerator.h is found, otherwise use the new ATen/CUDAGeneratorImpl.h, due to breaking change in https://github.com/pytorch/pytorch/pull/36026 
generator_flag = []
torch_dir = torch.__path__[0]
if os.path.exists(os.path.join(torch_dir, 'include', 'ATen', 'CUDAGenerator.h')):
    generator_flag = ['-DOLD_GENERATOR']

286
287
288
289
290
if "--fast_multihead_attn" in sys.argv:
    from torch.utils.cpp_extension import CUDAExtension
    sys.argv.remove("--fast_multihead_attn")

    from torch.utils.cpp_extension import BuildExtension
291
    cmdclass['build_ext'] = BuildExtension.with_options(use_ninja=False)
292
293
294
295
296
297
298

    if torch.utils.cpp_extension.CUDA_HOME is None:
        raise RuntimeError("--fast_multihead_attn was requested, but nvcc was not found.  Are you sure your environment has nvcc available?  If you're installing within a container from https://hub.docker.com/r/pytorch/pytorch, only images whose names contain 'devel' will provide nvcc.")
    else:
        subprocess.run(["git", "submodule", "update", "--init", "apex/contrib/csrc/multihead_attn/cutlass"])
        ext_modules.append(
            CUDAExtension(name='fast_self_multihead_attn',
299
                          sources=['apex/contrib/csrc/multihead_attn/self_multihead_attn.cpp',
300
                                   'apex/contrib/csrc/multihead_attn/self_multihead_attn_cuda.cu'],
ptrblck's avatar
ptrblck committed
301
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros + generator_flag,
302
                                              'nvcc':['-O3',
303
                                                      '-gencode', 'arch=compute_70,code=sm_70',
304
305
306
307
308
                                                      '-I./apex/contrib/csrc/multihead_attn/cutlass/',
                                                      '-U__CUDA_NO_HALF_OPERATORS__',
                                                      '-U__CUDA_NO_HALF_CONVERSIONS__',
                                                      '--expt-relaxed-constexpr',
                                                      '--expt-extended-lambda',
ptrblck's avatar
ptrblck committed
309
                                                      '--use_fast_math'] + version_dependent_macros + generator_flag}))
310
311
        ext_modules.append(
            CUDAExtension(name='fast_self_multihead_attn_norm_add',
312
                          sources=['apex/contrib/csrc/multihead_attn/self_multihead_attn_norm_add.cpp',
313
                                   'apex/contrib/csrc/multihead_attn/self_multihead_attn_norm_add_cuda.cu'],
ptrblck's avatar
ptrblck committed
314
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros + generator_flag,
315
                                              'nvcc':['-O3',
316
                                                      '-gencode', 'arch=compute_70,code=sm_70',
317
318
319
320
321
                                                      '-I./apex/contrib/csrc/multihead_attn/cutlass/',
                                                      '-U__CUDA_NO_HALF_OPERATORS__',
                                                      '-U__CUDA_NO_HALF_CONVERSIONS__',
                                                      '--expt-relaxed-constexpr',
                                                      '--expt-extended-lambda',
ptrblck's avatar
ptrblck committed
322
                                                      '--use_fast_math'] + version_dependent_macros + generator_flag}))
323
324
        ext_modules.append(
            CUDAExtension(name='fast_encdec_multihead_attn',
325
                          sources=['apex/contrib/csrc/multihead_attn/encdec_multihead_attn.cpp',
326
                                   'apex/contrib/csrc/multihead_attn/encdec_multihead_attn_cuda.cu'],
ptrblck's avatar
ptrblck committed
327
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros + generator_flag,
328
                                              'nvcc':['-O3',
329
                                                      '-gencode', 'arch=compute_70,code=sm_70',
330
331
332
333
334
                                                      '-I./apex/contrib/csrc/multihead_attn/cutlass/',
                                                      '-U__CUDA_NO_HALF_OPERATORS__',
                                                      '-U__CUDA_NO_HALF_CONVERSIONS__',
                                                      '--expt-relaxed-constexpr',
                                                      '--expt-extended-lambda',
ptrblck's avatar
ptrblck committed
335
                                                      '--use_fast_math'] + version_dependent_macros + generator_flag}))
336
337
        ext_modules.append(
            CUDAExtension(name='fast_encdec_multihead_attn_norm_add',
338
                          sources=['apex/contrib/csrc/multihead_attn/encdec_multihead_attn_norm_add.cpp',
339
                                   'apex/contrib/csrc/multihead_attn/encdec_multihead_attn_norm_add_cuda.cu'],
ptrblck's avatar
ptrblck committed
340
                          extra_compile_args={'cxx': ['-O3',] + version_dependent_macros + generator_flag,
341
                                              'nvcc':['-O3',
342
                                                      '-gencode', 'arch=compute_70,code=sm_70',
343
344
345
346
347
                                                      '-I./apex/contrib/csrc/multihead_attn/cutlass/',
                                                      '-U__CUDA_NO_HALF_OPERATORS__',
                                                      '-U__CUDA_NO_HALF_CONVERSIONS__',
                                                      '--expt-relaxed-constexpr',
                                                      '--expt-extended-lambda',
ptrblck's avatar
ptrblck committed
348
                                                      '--use_fast_math'] + version_dependent_macros + generator_flag}))
349

Christian Sarofeen's avatar
Christian Sarofeen committed
350
setup(
351
352
    name='apex',
    version='0.1',
353
354
355
356
    packages=find_packages(exclude=('build',
                                    'csrc',
                                    'include',
                                    'tests',
357
358
359
360
361
                                    'dist',
                                    'docs',
                                    'tests',
                                    'examples',
                                    'apex.egg-info',)),
Christian Sarofeen's avatar
Christian Sarofeen committed
362
    description='PyTorch Extensions written by NVIDIA',
jjsjann123's avatar
jjsjann123 committed
363
364
    ext_modules=ext_modules,
    cmdclass=cmdclass,
ptrblck's avatar
ptrblck committed
365
    extras_require=extras,
Christian Sarofeen's avatar
Christian Sarofeen committed
366
)