"docs/vscode:/vscode.git/clone" did not exist on "f855756961396c382826ba8b8078a081a7363fd8"
setup.py 18.6 KB
Newer Older
1
2
import glob
import os
3
import platform
Rinat Shigapov's avatar
Rinat Shigapov committed
4
import re
5
from pkg_resources import DistributionNotFound, get_distribution, parse_version
6
from setuptools import find_packages, setup
Yuanhao Zhu's avatar
Yuanhao Zhu committed
7
8
9
10
11
12
13

EXT_TYPE = ''
try:
    import torch
    if torch.__version__ == 'parrots':
        from parrots.utils.build_extension import BuildExtension
        EXT_TYPE = 'parrots'
14
15
16
17
    elif (hasattr(torch, 'is_mlu_available') and torch.is_mlu_available()) or \
            os.getenv('FORCE_MLU', '0') == '1':
        from torch_mlu.utils.cpp_extension import BuildExtension
        EXT_TYPE = 'pytorch'
Yuanhao Zhu's avatar
Yuanhao Zhu committed
18
19
20
    else:
        from torch.utils.cpp_extension import BuildExtension
        EXT_TYPE = 'pytorch'
21
    cmd_class = {'build_ext': BuildExtension}
Yuanhao Zhu's avatar
Yuanhao Zhu committed
22
except ModuleNotFoundError:
23
    cmd_class = {}
Yuanhao Zhu's avatar
Yuanhao Zhu committed
24
    print('Skip building ext ops due to the absence of torch.')
25

Rinat Shigapov's avatar
Rinat Shigapov committed
26
27

def choose_requirement(primary, secondary):
Kai Chen's avatar
Kai Chen committed
28
29
    """If some version of primary requirement installed, return primary, else
    return secondary."""
Rinat Shigapov's avatar
Rinat Shigapov committed
30
31
32
33
34
35
36
37
38
    try:
        name = re.split(r'[!<>=]', primary)[0]
        get_distribution(name)
    except DistributionNotFound:
        return secondary

    return str(primary)


39
40
def get_version():
    version_file = 'mmcv/version.py'
41
    with open(version_file, encoding='utf-8') as f:
42
43
44
45
        exec(compile(f.read(), version_file, 'exec'))
    return locals()['__version__']


46
def parse_requirements(fname='requirements/runtime.txt', with_version=True):
Kai Chen's avatar
Kai Chen committed
47
    """Parse the package dependencies listed in a requirements file but strips
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
    specific versioning information.

    Args:
        fname (str): path to requirements file
        with_version (bool, default=False): if True include version specs

    Returns:
        List[str]: list of requirements items

    CommandLine:
        python -c "import setup; print(setup.parse_requirements())"
    """
    import sys
    from os.path import exists
    require_fpath = fname

    def parse_line(line):
Kai Chen's avatar
Kai Chen committed
65
        """Parse information from a line in a requirements text file."""
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
        if line.startswith('-r '):
            # Allow specifying requirements in other files
            target = line.split(' ')[1]
            for info in parse_require_file(target):
                yield info
        else:
            info = {'line': line}
            if line.startswith('-e '):
                info['package'] = line.split('#egg=')[1]
            else:
                # Remove versioning from the package
                pat = '(' + '|'.join(['>=', '==', '>']) + ')'
                parts = re.split(pat, line, maxsplit=1)
                parts = [p.strip() for p in parts]

                info['package'] = parts[0]
                if len(parts) > 1:
                    op, rest = parts[1:]
                    if ';' in rest:
                        # Handle platform specific dependencies
                        # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies
                        version, platform_deps = map(str.strip,
                                                     rest.split(';'))
                        info['platform_deps'] = platform_deps
                    else:
                        version = rest  # NOQA
                    info['version'] = (op, version)
            yield info

    def parse_require_file(fpath):
96
        with open(fpath) as f:
97
98
99
            for line in f.readlines():
                line = line.strip()
                if line and not line.startswith('#'):
100
                    yield from parse_line(line)
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

    def gen_packages_items():
        if exists(require_fpath):
            for info in parse_require_file(require_fpath):
                parts = [info['package']]
                if with_version and 'version' in info:
                    parts.extend(info['version'])
                if not sys.version.startswith('3.4'):
                    # apparently package_deps are broken in 3.4
                    platform_deps = info.get('platform_deps')
                    if platform_deps is not None:
                        parts.append(';' + platform_deps)
                item = ''.join(parts)
                yield item

    packages = list(gen_packages_items())
    return packages


install_requires = parse_requirements()
121
122
123
124
125
126
127
128
129
130
131
132
133
134

try:
    # OpenCV installed via conda.
    import cv2  # NOQA: F401
    major, minor, *rest = cv2.__version__.split('.')
    if int(major) < 3:
        raise RuntimeError(
            f'OpenCV >=3 is required but {cv2.__version__} is installed')
except ImportError:
    # If first not installed install second package
    CHOOSE_INSTALL_REQUIRES = [('opencv-python-headless>=3',
                                'opencv-python>=3')]
    for main, secondary in CHOOSE_INSTALL_REQUIRES:
        install_requires.append(choose_requirement(main, secondary))
135

Kai Chen's avatar
Kai Chen committed
136

137
138
139
def get_extensions():
    extensions = []

Zaida Zhou's avatar
Zaida Zhou committed
140
    if os.getenv('MMCV_WITH_OPS', '1') == '0':
141
142
        return extensions

Yuanhao Zhu's avatar
Yuanhao Zhu committed
143
144
145
    if EXT_TYPE == 'parrots':
        ext_name = 'mmcv._ext'
        from parrots.utils.build_extension import Extension
146

147
148
149
        # new parrots op impl do not use MMCV_USE_PARROTS
        # define_macros = [('MMCV_USE_PARROTS', None)]
        define_macros = []
150
151
        include_dirs = []
        op_files = glob.glob('./mmcv/ops/csrc/pytorch/cuda/*.cu') +\
152
            glob.glob('./mmcv/ops/csrc/pytorch/cpu/*.cpp') +\
153
            glob.glob('./mmcv/ops/csrc/parrots/*.cpp')
154
155
        include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common'))
        include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common/cuda'))
CokeDong's avatar
CokeDong committed
156
157
158
        op_files.remove('./mmcv/ops/csrc/pytorch/cuda/iou3d_cuda.cu')
        op_files.remove('./mmcv/ops/csrc/pytorch/cpu/bbox_overlaps_cpu.cpp')
        op_files.remove('./mmcv/ops/csrc/pytorch/cuda/bias_act_cuda.cu')
Yuanhao Zhu's avatar
Yuanhao Zhu committed
159
        cuda_args = os.getenv('MMCV_CUDA_ARGS')
160
        extra_compile_args = {
161
162
            'nvcc': [cuda_args, '-std=c++14'] if cuda_args else ['-std=c++14'],
            'cxx': ['-std=c++14'],
163
164
165
166
167
168
169
170
        }
        if torch.cuda.is_available() or os.getenv('FORCE_CUDA', '0') == '1':
            define_macros += [('MMCV_WITH_CUDA', None)]
            extra_compile_args['nvcc'] += [
                '-D__CUDA_NO_HALF_OPERATORS__',
                '-D__CUDA_NO_HALF_CONVERSIONS__',
                '-D__CUDA_NO_HALF2_OPERATORS__',
            ]
Yuanhao Zhu's avatar
Yuanhao Zhu committed
171
172
173
        ext_ops = Extension(
            name=ext_name,
            sources=op_files,
174
            include_dirs=include_dirs,
Yuanhao Zhu's avatar
Yuanhao Zhu committed
175
            define_macros=define_macros,
176
177
178
            extra_compile_args=extra_compile_args,
            cuda=True,
            pytorch=True)
Yuanhao Zhu's avatar
Yuanhao Zhu committed
179
180
        extensions.append(ext_ops)
    elif EXT_TYPE == 'pytorch':
181
        ext_name = 'mmcv._ext'
182
183
        from torch.utils.cpp_extension import CppExtension, CUDAExtension

Yuanhao Zhu's avatar
Yuanhao Zhu committed
184
        # prevent ninja from using too many resources
lizz's avatar
lizz committed
185
186
187
188
189
190
191
192
        try:
            import psutil
            num_cpu = len(psutil.Process().cpu_affinity())
            cpu_use = max(4, num_cpu - 1)
        except (ModuleNotFoundError, AttributeError):
            cpu_use = 4

        os.environ.setdefault('MAX_JOBS', str(cpu_use))
Yuanhao Zhu's avatar
Yuanhao Zhu committed
193
        define_macros = []
194
195
196
197
198
199
200
201
202

        # Before PyTorch1.8.0, when compiling CUDA code, `cxx` is a
        # required key passed to PyTorch. Even if there is no flag passed
        # to cxx, users also need to pass an empty list to PyTorch.
        # Since PyTorch1.8.0, it has a default value so users do not need
        # to pass an empty list anymore.
        # More details at https://github.com/pytorch/pytorch/pull/45956
        extra_compile_args = {'cxx': []}

203
204
        if platform.system() != 'Windows':
            extra_compile_args['cxx'] = ['-std=c++14']
205
206
207
208
209
210
        else:
            # TODO: In Windows, C++17 is chosen to compile extensions in
            # PyTorch2.0 , but a compile error will be reported.
            # As a temporary solution, force the use of C++14.
            if parse_version(torch.__version__) >= parse_version('2.0.0'):
                extra_compile_args['cxx'] = ['/std:c++14']
211

212
        include_dirs = []
Yuanhao Zhu's avatar
Yuanhao Zhu committed
213

214
        extra_objects = []
bdf's avatar
bdf committed
215
        extra_link_args = []
216
        is_rocm_pytorch = False
217
        try:
218
219
220
            from torch.utils.cpp_extension import ROCM_HOME
            is_rocm_pytorch = True if ((torch.version.hip is not None) and
                                       (ROCM_HOME is not None)) else False
221
222
        except ImportError:
            pass
223

q.yao's avatar
q.yao committed
224
225
226
        if is_rocm_pytorch or torch.cuda.is_available() or os.getenv(
                'FORCE_CUDA', '0') == '1':
            if is_rocm_pytorch:
Zachary Streeter's avatar
Zachary Streeter committed
227
                define_macros += [('MMCV_WITH_HIP', None)]
Yuanhao Zhu's avatar
Yuanhao Zhu committed
228
            define_macros += [('MMCV_WITH_CUDA', None)]
zhuyuanhao's avatar
zhuyuanhao committed
229
            cuda_args = os.getenv('MMCV_CUDA_ARGS')
Yuanhao Zhu's avatar
Yuanhao Zhu committed
230
            extra_compile_args['nvcc'] = [cuda_args] if cuda_args else []
231
            op_files = glob.glob('./mmcv/ops/csrc/pytorch/*.cpp') + \
232
233
234
                glob.glob('./mmcv/ops/csrc/pytorch/cpu/*.cpp') + \
                glob.glob('./mmcv/ops/csrc/pytorch/cuda/*.cu') + \
                glob.glob('./mmcv/ops/csrc/pytorch/cuda/*.cpp')
Yuanhao Zhu's avatar
Yuanhao Zhu committed
235
            extension = CUDAExtension
Zachary Streeter's avatar
Zachary Streeter committed
236
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/pytorch'))
237
238
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common'))
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common/cuda'))
239
240
241
242
        elif (hasattr(torch, 'is_mlu_available') and
                torch.is_mlu_available()) or \
                os.getenv('FORCE_MLU', '0') == '1':
            from torch_mlu.utils.cpp_extension import MLUExtension
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308

            def get_mluops_version(file_path):
                with open(file_path) as f:
                    for line in f:
                        if re.search('MLUOP_MAJOR', line):
                            major = line.strip().split(' ')[2]
                        if re.search('MLUOP_MINOR', line):
                            minor = line.strip().split(' ')[2]
                        if re.search('MLUOP_PATCHLEVEL', line):
                            patchlevel = line.strip().split(' ')[2]
                mluops_version = f'v{major}.{minor}.{patchlevel}'
                return mluops_version

            mmcv_mluops_version = get_mluops_version(
                './mmcv/ops/csrc/pytorch/mlu/mlu_common_helper.h')
            mlu_ops_path = os.getenv('MMCV_MLU_OPS_PATH')
            if mlu_ops_path:
                exists_mluops_version = get_mluops_version(
                    mlu_ops_path + '/bangc-ops/mlu_op.h')
                if exists_mluops_version != mmcv_mluops_version:
                    print('the version of mlu-ops provided is %s,'
                          ' while %s is needed.' %
                          (exists_mluops_version, mmcv_mluops_version))
                    exit()
                try:
                    if os.path.exists('mlu-ops'):
                        if os.path.islink('mlu-ops'):
                            os.remove('mlu-ops')
                            os.symlink(mlu_ops_path, 'mlu-ops')
                        elif os.path.abspath('mlu-ops') != mlu_ops_path:
                            os.symlink(mlu_ops_path, 'mlu-ops')
                    else:
                        os.symlink(mlu_ops_path, 'mlu-ops')
                except Exception:
                    raise FileExistsError(
                        'mlu-ops already exists, please move it out,'
                        'or rename or remove it.')
            else:
                if not os.path.exists('mlu-ops'):
                    import requests
                    mluops_url = 'https://github.com/Cambricon/mlu-ops/' + \
                        'archive/refs/tags/' + mmcv_mluops_version + '.zip'
                    req = requests.get(mluops_url)
                    with open('./mlu-ops.zip', 'wb') as f:
                        try:
                            f.write(req.content)
                        except Exception:
                            raise ImportError('failed to download mlu-ops')

                    from zipfile import BadZipFile, ZipFile
                    with ZipFile('./mlu-ops.zip', 'r') as archive:
                        try:
                            archive.extractall()
                            dir_name = archive.namelist()[0].split('/')[0]
                            os.rename(dir_name, 'mlu-ops')
                        except BadZipFile:
                            print('invalid mlu-ops.zip file')
                else:
                    exists_mluops_version = get_mluops_version(
                        './mlu-ops/bangc-ops/mlu_op.h')
                    if exists_mluops_version != mmcv_mluops_version:
                        print('the version of provided mlu-ops is %s,'
                              ' while %s is needed.' %
                              (exists_mluops_version, mmcv_mluops_version))
                        exit()

309
            define_macros += [('MMCV_WITH_MLU', None)]
310
311
312
313
314
315
316
317
318
            mlu_args = os.getenv('MMCV_MLU_ARGS', '-DNDEBUG ')
            mluops_includes = []
            mluops_includes.append('-I' +
                                   os.path.abspath('./mlu-ops/bangc-ops'))
            mluops_includes.append(
                '-I' + os.path.abspath('./mlu-ops/bangc-ops/kernels'))
            extra_compile_args['cncc'] = [mlu_args] + \
                mluops_includes if mlu_args else mluops_includes
            extra_compile_args['cxx'] += ['-fno-gnu-unique']
319
320
            op_files = glob.glob('./mmcv/ops/csrc/pytorch/*.cpp') + \
                glob.glob('./mmcv/ops/csrc/pytorch/cpu/*.cpp') + \
321
                glob.glob('./mmcv/ops/csrc/pytorch/mlu/*.cpp') + \
322
323
324
325
326
327
328
                glob.glob('./mmcv/ops/csrc/common/mlu/*.mlu') + \
                glob.glob(
                    './mlu-ops/bangc-ops/core/**/*.cpp', recursive=True) + \
                glob.glob(
                    './mlu-ops/bangc-ops/kernels/**/*.cpp', recursive=True) + \
                glob.glob(
                    './mlu-ops/bangc-ops/kernels/**/*.mlu', recursive=True)
bdf's avatar
bdf committed
329
330
331
332
333
            extra_link_args = [
                '-Wl,--whole-archive',
                './mlu-ops/bangc-ops/kernels/kernel_wrapper/lib/libextops.a',
                '-Wl,--no-whole-archive'
            ]
334
335
336
            extension = MLUExtension
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common'))
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common/mlu'))
337
            include_dirs.append(os.path.abspath('./mlu-ops/bangc-ops'))
Zaida Zhou's avatar
Zaida Zhou committed
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
        elif (hasattr(torch.backends, 'mps')
              and torch.backends.mps.is_available()) or os.getenv(
                  'FORCE_MPS', '0') == '1':
            # objc compiler support
            from distutils.unixccompiler import UnixCCompiler
            if '.mm' not in UnixCCompiler.src_extensions:
                UnixCCompiler.src_extensions.append('.mm')
                UnixCCompiler.language_map['.mm'] = 'objc'

            define_macros += [('MMCV_WITH_MPS', None)]
            extra_compile_args = {}
            extra_compile_args['cxx'] = ['-Wall', '-std=c++17']
            extra_compile_args['cxx'] += [
                '-framework', 'Metal', '-framework', 'Foundation'
            ]
            extra_compile_args['cxx'] += ['-ObjC++']
            # src
            op_files = glob.glob('./mmcv/ops/csrc/pytorch/*.cpp') + \
                glob.glob('./mmcv/ops/csrc/pytorch/cpu/*.cpp') + \
                glob.glob('./mmcv/ops/csrc/common/mps/*.mm') + \
                glob.glob('./mmcv/ops/csrc/pytorch/mps/*.mm')
            extension = CppExtension
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common'))
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common/mps'))
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
        elif (os.getenv('FORCE_NPU', '0') == '1'):
            print(f'Compiling {ext_name} only with CPU and NPU')
            try:
                from torch_npu.utils.cpp_extension import NpuExtension
                define_macros += [('MMCV_WITH_NPU', None)]
                extension = NpuExtension
            except Exception:
                raise ImportError('can not find any torch_npu')
            # src
            op_files = glob.glob('./mmcv/ops/csrc/pytorch/*.cpp') + \
                glob.glob('./mmcv/ops/csrc/pytorch/cpu/*.cpp') + \
                glob.glob('./mmcv/ops/csrc/common/npu/*.cpp') + \
                glob.glob('./mmcv/ops/csrc/pytorch/npu/*.cpp')
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common'))
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common/npu'))
377
        else:
378
            print(f'Compiling {ext_name} only with CPU')
379
380
            op_files = glob.glob('./mmcv/ops/csrc/pytorch/*.cpp') + \
                glob.glob('./mmcv/ops/csrc/pytorch/cpu/*.cpp')
Yuanhao Zhu's avatar
Yuanhao Zhu committed
381
            extension = CppExtension
382
            include_dirs.append(os.path.abspath('./mmcv/ops/csrc/common'))
Yuanhao Zhu's avatar
Yuanhao Zhu committed
383

384
385
386
387
388
389
390
391
        # Since the PR (https://github.com/open-mmlab/mmcv/pull/1463) uses
        # c++14 features, the argument ['std=c++14'] must be added here.
        # However, in the windows environment, some standard libraries
        # will depend on c++17 or higher. In fact, for the windows
        # environment, the compiler will choose the appropriate compiler
        # to compile those cpp files, so there is no need to add the
        # argument
        if 'nvcc' in extra_compile_args and platform.system() != 'Windows':
392
393
            extra_compile_args['nvcc'] += ['-std=c++14']

Yuanhao Zhu's avatar
Yuanhao Zhu committed
394
395
396
        ext_ops = extension(
            name=ext_name,
            sources=op_files,
397
            include_dirs=include_dirs,
Yuanhao Zhu's avatar
Yuanhao Zhu committed
398
            define_macros=define_macros,
399
            extra_objects=extra_objects,
bdf's avatar
bdf committed
400
401
            extra_compile_args=extra_compile_args,
            extra_link_args=extra_link_args)
Yuanhao Zhu's avatar
Yuanhao Zhu committed
402
        extensions.append(ext_ops)
403
404
    return extensions

405

406
setup(
Zaida Zhou's avatar
Zaida Zhou committed
407
    name='mmcv' if os.getenv('MMCV_WITH_OPS', '1') == '1' else 'mmcv-lite',
408
    version=get_version(),
409
    description='OpenMMLab Computer Vision Foundation',
410
411
    keywords='computer vision',
    packages=find_packages(),
412
    include_package_data=True,
413
414
    classifiers=[
        'Development Status :: 4 - Beta',
Kai Chen's avatar
Kai Chen committed
415
        'License :: OSI Approved :: Apache Software License',
416
417
        'Operating System :: OS Independent',
        'Programming Language :: Python :: 3',
418
        'Programming Language :: Python :: 3.7',
419
        'Programming Language :: Python :: 3.8',
420
        'Programming Language :: Python :: 3.9',
Zaida Zhou's avatar
Zaida Zhou committed
421
        'Programming Language :: Python :: 3.10',
422
423
424
        'Topic :: Utilities',
    ],
    url='https://github.com/open-mmlab/mmcv',
Zaida Zhou's avatar
Zaida Zhou committed
425
    author='MMCV Contributors',
Kai Chen's avatar
Kai Chen committed
426
    author_email='openmmlab@gmail.com',
427
    install_requires=install_requires,
428
429
430
431
432
433
    extras_require={
        'all': parse_requirements('requirements.txt'),
        'tests': parse_requirements('requirements/test.txt'),
        'build': parse_requirements('requirements/build.txt'),
        'optional': parse_requirements('requirements/optional.txt'),
    },
434
    python_requires='>=3.7',
435
    ext_modules=get_extensions(),
436
    cmdclass=cmd_class,
437
    zip_safe=False)