pip support added

Signed-off-by: Evelina Bakhturina <ebakhturina@nvidia.com>

.gitignore

 __pycache__
+
+# Distribution / packaging
+build/
+dist/
+*.egg-info/
\ No newline at end of file

megatron/__init__.py

@@ -13,14 +13,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import torch
+import os
+
+from .package_info import (
+    __contact_emails__,
+    __contact_names__,
+    __description__,
+    __url__,
+    __download_url__,
+    __keywords__,
+    __license__,
+    __package_name__,
+    __version__,
+)
 
-from .global_vars import get_args
-from .global_vars import get_tokenizer
-from .global_vars import get_tensorboard_writer
-from .global_vars import get_adlr_autoresume
-from .global_vars import get_timers
+if "MEGATRON_PACKAGE_BUILDING" not in os.environ:
+    from .global_vars import get_args
+    from .global_vars import get_tokenizer
+    from .global_vars import get_tensorboard_writer
+    from .global_vars import get_adlr_autoresume
+    from .global_vars import get_timers
 
+import torch
 
 def print_rank_0(message):
     """If distributed is initialized print only on rank 0."""

megatron/package_info.py

+# ! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+MAJOR = 0
+MINOR = 0
+PATCH = 1
+
+# Use the following formatting: (major, minor, patch)
+VERSION = (MAJOR, MINOR, PATCH)
+
+__version__ = '.'.join(map(str, VERSION[:3]))
+__package_name__ = 'megatron_lm'
+__contact_names__ = 'NVIDIA'
+__contact_emails__ = 'ekmb.new@gmail.com'
+__url__ = 'https://github.com/NVIDIA/Megatron-LM'
+__download_url__ = 'https://github.com/NVIDIA/Megatron-LM/releases'
+__description__ = 'Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism.'
+__license__ = 'Apache2'
+__keywords__ = 'deep learning, Megatron, gpu, NLP, nvidia, pytorch, torch, language'
+

requirements.txt

+setuptools
+nltk
\ No newline at end of file

setup.py

+# ! /usr/bin/python
+# -*- coding: utf-8 -*-
+
+# Copyright 2020 NVIDIA. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+"""Setup for pip package."""
+
+import codecs
+import os
+import subprocess
+import sys
+from distutils import cmd as distutils_cmd
+from distutils import log as distutils_log
+from itertools import chain
+
+import setuptools
+
+import sys
+if sys.version_info < (3,):
+    raise Exception("Python 2 is not supported by Megatron.")
+
+def is_build_action():
+    if len(sys.argv) <= 1:
+        return False
+
+    BUILD_TOKENS = ["egg_info", "dist", "bdist", "sdist", "install", "build", "develop", "style"]
+
+    if any([sys.argv[1].startswith(x) for x in BUILD_TOKENS]):
+        return True
+    else:
+        return False
+
+
+if is_build_action():
+    os.environ['MEGATRON_PACKAGE_BUILDING'] = 'True'
+
+from megatron.package_info import (
+    __contact_emails__,
+    __contact_names__,
+    __description__,
+    __url__,
+    __download_url__,
+    __keywords__,
+    __license__,
+    __package_name__,
+    __version__,
+)
+
+###############################################################################
+#                             Dependency Loading                              #
+# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% #
+
+
+def req_file(filename):
+    with open(filename) as f:
+        content = f.readlines()
+    # you may also want to remove whitespace characters
+    # Example: `\n` at the end of each line
+    return [x.strip() for x in content]
+
+
+install_requires = req_file("requirements.txt")
+
+setuptools.setup(
+    name=__package_name__ + '_test',
+    # Versions should comply with PEP440.  For a discussion on single-sourcing
+    # the version across setup.py and the project code, see
+    # https://packaging.python.org/en/latest/single_source_version.html
+    version=__version__,
+    description=__description__,
+    # The project's main homepage.
+    url=__url__,
+    # Author details
+    author=__contact_names__,
+    author_email=__contact_emails__,
+    # maintainer Details
+    maintainer=__contact_names__,
+    maintainer_email=__contact_emails__,
+    # The licence under which the project is released
+    license=__license__,
+    classifiers=[
+        # How mature is this project? Common values are
+        #  1 - Planning
+        #  2 - Pre-Alpha
+        #  3 - Alpha
+        #  4 - Beta
+        #  5 - Production/Stable
+        #  6 - Mature
+        #  7 - Inactive
+        'Development Status :: 4 - Beta',
+        # Indicate who your project is intended for
+        'Intended Audience :: Developers',
+        'Intended Audience :: Science/Research',
+        'Intended Audience :: Information Technology',
+        # Indicate what your project relates to
+        'Topic :: Scientific/Engineering',
+        'Topic :: Scientific/Engineering :: Mathematics',
+        'Topic :: Scientific/Engineering :: Image Recognition',
+        'Topic :: Scientific/Engineering :: Artificial Intelligence',
+        'Topic :: Software Development :: Libraries',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Topic :: Utilities',
+        # Pick your license as you wish (should match "license" above)
+        'License :: OSI Approved :: Apache Software License',
+        # Supported python versions
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        # Additional Setting
+        'Environment :: Console',
+        'Natural Language :: English',
+        'Operating System :: OS Independent',
+    ],
+    packages=setuptools.find_packages(),
+    install_requires=install_requires,
+    # Add in any packaged data.
+    include_package_data=True,
+    zip_safe=False,
+    # PyPI package information.
+    keywords=__keywords__,
+)