Refactor pip installation logic for supporting uninstall

deployment/docker/Dockerfile.build.base

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
+
+LABEL maintainer='Microsoft NNI Team<nni@microsoft.com>'
+
+ENV HADOOP_VERSION=2.7.2
+LABEL HADOOP_VERSION=2.7.2
+
+RUN DEBIAN_FRONTEND=noninteractive && \
+    apt-get -y update && \
+    apt-get -y install sudo \
+    apt-utils \
+    git \
+    curl \
+    vim \
+    unzip \
+    wget \
+    build-essential \ 
+    cmake \
+    libopenblas-dev \
+    automake \
+    openjdk-8-jdk \
+    openssh-client \
+    openssh-server \
+    lsof \
+    python3.5 \
+    python3-dev \
+    python3-pip \
+    python3-tk \
+    libcupti-dev && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# numpy 1.14.3  scipy 1.1.0 
+RUN pip3 --no-cache-dir install \
+    numpy==1.14.3 scipy==1.1.0 
+
+#
+#Install hadoop
+#
+RUN wget -qO- http://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz | \
+    tar xz -C /usr/local && \
+    mv /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop
+
+#
+#Install NNI
+#
+RUN pip3 install -v --user git+https://github.com/Microsoft/nni.git@v0.2
+
+ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \
+    HADOOP_INSTALL=/usr/local/hadoop \
+    NVIDIA_VISIBLE_DEVICES=all
+
+ENV HADOOP_PREFIX=${HADOOP_INSTALL} \
+    HADOOP_BIN_DIR=${HADOOP_INSTALL}/bin \
+    HADOOP_SBIN_DIR=${HADOOP_INSTALL}/sbin \
+    HADOOP_HDFS_HOME=${HADOOP_INSTALL} \
+    HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_INSTALL}/lib/native \
+    HADOOP_OPTS="-Djava.library.path=${HADOOP_INSTALL}/lib/native"
+
+ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin:/sbin:/bin:${HADOOP_BIN_DIR}:${HADOOP_SBIN_DIR} \
+    LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/targets/x86_64-linux/lib/stubs:${JAVA_HOME}/jre/lib/amd64/server
+
+WORKDIR /root

deployment/pypi/.gitignore

+README.md
+nni_pkg/
+nni_annotation/
+nnicmd/
+dist/
+build/
+*.egg-info/
+node-*.tar.xz
+node-*/
\ No newline at end of file

deployment/pypi/Makefile

+## Colorful output
+_INFO := $(shell echo -e '\e[1;36m')
+_WARNING := $(shell echo -e '\e[1;33m')
+_END := $(shell echo -e '\e[0m')
+CWD := $(PWD)/
+
+.PHONY: build
+build:
+	python3 -m pip install --user --upgrade setuptools wheel
+	wget https://aka.ms/nodejs-download -O $(CWD)node-linux-x64.tar.xz
+	rm -rf $(CWD)node-linux-x64
+	mkdir $(CWD)node-linux-x64
+	tar xf $(CWD)node-linux-x64.tar.xz -C node-linux-x64 --strip-components 1
+	cp $(CWD)../../README.md $(CWD)
+	cd $(CWD)../../src/nni_manager && yarn && yarn build
+	cd $(CWD)../../src/webui && yarn && yarn build
+	rm -rf $(CWD)nni_pkg
+	cp -r $(CWD)../../src/nni_manager/dist $(CWD)nni_pkg
+	cp -r $(CWD)../../src/webui/build $(CWD)nni_pkg/static
+	cp $(CWD)../../src/nni_manager/package.json $(CWD)nni_pkg
+	cd $(CWD)nni_pkg && yarn --prod
+	rm -rf $(CWD)nnicmd
+	rm -rf $(CWD)nni_annotation
+	cp -r $(CWD)../../tools/nnicmd $(CWD)nnicmd
+	cp -r $(CWD)../../tools/nni_annotation $(CWD)nni_annotation
+	cd $(CWD) && python3 setup.py bdist_wheel
+	cd $(CWD)../../src/sdk/pynni && python3 setup.py bdist_wheel
+	cp -r $(CWD)../../src/sdk/pynni/dist/*.whl $(CWD)dist
+
+.PHONY: upload
+upload:
+	python3 -m pip install --user --upgrade twine
+	python3 -m twine upload dist/*
\ No newline at end of file

deployment/pypi/setup.py

+import setuptools
+from os import walk, path
+
+data_files = [('bin', ['node-linux-x64/bin/node'])]
+for (dirpath, dirnames, filenames) in walk('./nni_pkg'):
+    files = [path.normpath(path.join(dirpath, filename)) for filename in filenames]
+    data_files.append((path.normpath(dirpath), files))
+
+with open('../../README.md', 'r') as fh:
+    long_description = fh.read()
+
+setuptools.setup(
+    name = 'nni-pkg',
+    version = '0.3',
+    author = 'Microsoft NNI team',
+    author_email = 'nni@microsoft.com',
+    description = 'Neural Network Intelligence package',
+    long_description = long_description,
+    long_description_content_type = 'text/markdown',
+    license = 'MIT',
+    url = 'https://github.com/Microsoft/nni',
+    packages = setuptools.find_packages(),
+    python_requires = '>=3.5',
+    install_requires = [
+        'nni'
+    ],
+    classifiers = [
+        'Programming Language :: Python :: 3',
+        'License :: OSI Approved :: MIT License',
+        'Operating System :: POSIX :: Linux'
+    ],
+    data_files = data_files,
+    entry_points = {
+        'console_scripts' : [
+            'nnictl = nnicmd.nnictl:parse_args'
+        ]
+    }
+)
\ No newline at end of file

src/sdk/pynni/.gitignore

+dist/
+build/
+*.egg-info/
\ No newline at end of file

tools/nnicmd/launcher.py

@@ -35,6 +35,8 @@ from .constants import *
 import time
 import random
 import string
+from os import path
+import site
 
 def get_log_path(config_file_name):
     '''generate stdout and stderr log path'''
@@ -65,7 +67,7 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None
 
     print_normal('Starting restful server...')
     manager = os.environ.get('NNI_MANAGER', 'nnimanager')
-    cmds = [manager, '--port', str(port), '--mode', platform, '--start_mode', mode]
+    cmds = ['node', path.join(site.USER_BASE, 'nni_pkg', 'main.js'), '--port', str(port), '--mode', platform, '--start_mode', mode]
     if mode == 'resume':
         cmds += ['--experiment_id', experiment_id]
     stdout_full_path, stderr_full_path = get_log_path(config_file_name)
@@ -76,7 +78,7 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None
     log_header = LOG_HEADER % str(time_now)
     stdout_file.write(log_header)
     stderr_file.write(log_header)
-    process = Popen(cmds, stdout=stdout_file, stderr=stderr_file)
+    process = Popen(cmds, cwd=path.join(site.USER_BASE, 'nni_pkg'), stdout=stdout_file, stderr=stderr_file)
     return process, str(time_now)
 
 def set_trial_config(experiment_config, port, config_file_name):