setup.py 4.7 KB
Newer Older
zbian's avatar
zbian committed
1
import os
2
import sys
3
from datetime import datetime
4
from typing import List
5

Jiarui Fang's avatar
Jiarui Fang committed
6
from setuptools import find_packages, setup
zbian's avatar
zbian committed
7

8
try:
9
10
    import torch  # noqa
    from torch.utils.cpp_extension import BuildExtension
11

12
    TORCH_AVAILABLE = True
13
except ImportError:
14
    TORCH_AVAILABLE = False
15

16
THIS_DIR = os.path.dirname(os.path.abspath(__file__))
17
BUILD_EXT = int(os.environ.get("BUILD_EXT", "0")) == 1
18
IS_NIGHTLY = int(os.environ.get("NIGHTLY", "0")) == 1
19

20
# we do not support windows currently
21
if sys.platform == "win32":
22
23
    raise RuntimeError("Windows is not supported yet. Please try again within the Windows Subsystem for Linux (WSL).")

24
25
26
27
28
29
30
31
32
33
34

def fetch_requirements(path) -> List[str]:
    """
    This function reads the requirements file.

    Args:
        path (str): the path to the requirements file.

    Returns:
        The lines in the requirements file.
    """
35
    with open(path, "r") as fd:
36
37
38
        return [r.strip() for r in fd.readlines()]


39
40
41
42
43
44
45
def fetch_readme() -> str:
    """
    This function reads the README.md file in the current directory.

    Returns:
        The lines in the README file.
    """
46
    with open("README.md", encoding="utf-8") as f:
ver217's avatar
ver217 committed
47
48
49
        return f.read()


50
51
52
53
54
55
56
57
def get_version() -> str:
    """
    This function reads the version.txt and generates the colossalai/version.py file.

    Returns:
        The library version stored in version.txt.
    """

58
59
    setup_file_path = os.path.abspath(__file__)
    project_path = os.path.dirname(setup_file_path)
60
61
    version_txt_path = os.path.join(project_path, "version.txt")
    version_py_path = os.path.join(project_path, "colossalai/version.py")
62
63

    with open(version_txt_path) as f:
64
        version = f.read().strip()
ver217's avatar
ver217 committed
65

66
    # write version into version.py
67
    with open(version_py_path, "w") as f:
68
69
        f.write(f"__version__ = '{version}'\n")
    return version
70
71


72
73
74
75
76
if BUILD_EXT:
    if not TORCH_AVAILABLE:
        raise ModuleNotFoundError(
            "[extension] PyTorch is not found while CUDA_EXT=1. You need to install PyTorch first in order to build CUDA extensions"
        )
77

78
    from extensions import ALL_EXTENSIONS
79

80
    op_names = []
81
    ext_modules = []
82

83
84
85
86
87
88
    for ext_cls in ALL_EXTENSIONS:
        ext = ext_cls()
        if ext.support_aot and ext.is_hardware_available():
            ext.assert_hardware_compatible()
            op_names.append(ext.name)
            ext_modules.append(ext.build_aot())
89

90
    # show log
91
92
93
94
95
96
97
    if len(ext_modules) == 0:
        raise RuntimeError("[extension] Could not find any kernel compatible with the current environment.")
    else:
        op_name_list = ", ".join(op_names)
        print(f"[extension] Building extensions{op_name_list}")
else:
    ext_modules = []
98

99
100
101
# always put not nightly branch as the if branch
# otherwise github will treat colossalai-nightly as the project name
# and it will mess up with the dependency graph insights
102
if not IS_NIGHTLY:
103
    version = get_version()
104
    package_name = "colossalai"
105
else:
106
    # use date as the nightly version
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    version = datetime.today().strftime("%Y.%m.%d")
    package_name = "colossalai-nightly"

setup(
    name=package_name,
    version=version,
    packages=find_packages(
        exclude=(
            "op_builder",
            "benchmark",
            "docker",
            "tests",
            "docs",
            "examples",
            "tests",
            "scripts",
            "requirements",
124
            "extensions",
125
            "*.egg-info",
126
        ),
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
    ),
    description="An integrated large-scale model training system with efficient parallelization techniques",
    long_description=fetch_readme(),
    long_description_content_type="text/markdown",
    license="Apache Software License 2.0",
    url="https://www.colossalai.org",
    project_urls={
        "Forum": "https://github.com/hpcaitech/ColossalAI/discussions",
        "Bug Tracker": "https://github.com/hpcaitech/ColossalAI/issues",
        "Examples": "https://github.com/hpcaitech/ColossalAI-Examples",
        "Documentation": "http://colossalai.readthedocs.io",
        "Github": "https://github.com/hpcaitech/ColossalAI",
    },
    ext_modules=ext_modules,
    cmdclass={"build_ext": BuildExtension} if ext_modules else {},
    install_requires=fetch_requirements("requirements/requirements.txt"),
    entry_points="""
144
        [console_scripts]
145
        colossalai=colossalai.cli:cli
146
147
148
149
150
151
152
153
154
155
156
    """,
    python_requires=">=3.6",
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: Apache Software License",
        "Environment :: GPU :: NVIDIA CUDA",
        "Topic :: Scientific/Engineering :: Artificial Intelligence",
        "Topic :: System :: Distributed Computing",
    ],
    package_data={
        "colossalai": [
157
            "kernel/extensions/csrc/**/*",
158
159
160
        ]
    },
)