Fix issues for pip installation

Makefile

 # Setting variables
 
-SHELL := /bin/bash
-PIP_INSTALL := pip3 install
-PIP_UNINSTALL := pip3 uninstall
+PIP_INSTALL := python3 -m pip install
+PIP_UNINSTALL := python3 -m pip uninstall
 
 ## Colorful output
 _INFO := $(shell echo -e '\e[1;36m')
@@ -12,18 +11,19 @@ _END := $(shell echo -e '\e[0m')
 ## Install directories
 ifeq ($(shell id -u), 0)  # is root
     _ROOT := 1
-    BIN_FOLDER ?= /usr/local/bin
-    NNI_PKG_FOLDER ?= /usr/local/nni_pkg
+    ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])')
     BASH_COMP_SCRIPT ?= /usr/share/bash-completion/completions/nnictl
 else  # is normal user
-    BIN_FOLDER ?= ${HOME}/.local/bin
-    NNI_PKG_FOLDER ?= ${HOME}/.local/nni_pkg
+    ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])')
     ifndef VIRTUAL_ENV
         PIP_MODE ?= --user
     endif
     BASH_COMP_SCRIPT ?= ${HOME}/.bash_completion.d/nnictl
 endif
 
+BIN_FOLDER ?= $(ROOT_FOLDER)/bin
+NNI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni_pkg
+
 ## Dependency information
 $(info $(_INFO) Installing dependencies, use local toolchain $(_END))
 NNI_NODE_TARBALL ?= /tmp/nni-node-linux-x64.tar.xz
@@ -39,33 +39,13 @@ NNI_YARN := PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn
 build:
 	#$(_INFO) Building NNI Manager $(_END)
 	cd src/nni_manager && $(NNI_YARN) && $(NNI_YARN) build
-	
 	#$(_INFO) Building WebUI $(_END)
 	cd src/webui && $(NNI_YARN) && $(NNI_YARN) build
-	
 	#$(_INFO) Building Python SDK $(_END)
 	cd src/sdk/pynni && python3 setup.py build
-	
 	#$(_INFO) Building nnictl $(_END)
 	cd tools && python3 setup.py build
 
-# Standard installation target
-# Must be invoked after building
-.PHONY: install
-install: install-python-modules
-install: install-node-modules
-install: install-scripts
-install:
-	#$(_INFO) Complete! You may want to add $(BIN_FOLDER) to your PATH environment $(_END)
-
-
-# Target for remote machine workers
-# Only installs core SDK module
-.PHONY: remote-machine-install
-remote-machine-install:
-	cd src/sdk/pynni && python3 setup.py install $(PIP_MODE)
-
-
 # All-in-one target for non-expert users
 # Installs NNI as well as its dependencies, and update bashrc to set PATH
 .PHONY: easy-install
@@ -74,30 +54,46 @@ easy-install: install-dependencies
 easy-install: build
 easy-install: install
 easy-install: update-bash-config
-
 easy-install:
 	#$(_INFO) Complete! $(_END)
 
+# All-in-one target for developer users
+# Install NNI as well as its dependencies, and update bashrc to set PATH
+.PHONY: dev-easy-install
+dev-easy-install: dev-check-perm
+dev-easy-install: install-dependencies
+dev-easy-install: build
+dev-easy-install: dev-install
+dev-easy-install: update-bash-config
+dev-easy-install:
+	#$(_INFO) Complete! $(_END)
 
-# Target for setup.py
-# Do not invoke this manually
-.PHONY: pip-install
-pip-install: install-dependencies
-pip-install: build
-pip-install: install-node-modules
-pip-install: install-scripts
-pip-install: update-bash-config
-
+# Standard installation target
+# Must be invoked after building
+.PHONY: install
+install: install-python-modules
+install: install-node-modules
+install: install-scripts
+install:
+	#$(_INFO) Complete! You may want to add $(BIN_FOLDER) to your PATH environment $(_END)
 
 # Target for NNI developers
 # Creates symlinks instead of copying files
 .PHONY: dev-install
-dev-install: check-dev-env
-dev-install: install-dev-modules
+dev-install: dev-install-python-modules
+dev-install: dev-install-node-modules
 dev-install: install-scripts
 dev-install:
 	#$(_INFO) Complete! You may want to add $(BIN_FOLDER) to your PATH environment $(_END)
 
+# Target for setup.py
+# Do not invoke this manually
+.PHONY: pip-install
+pip-install: install-dependencies
+pip-install: build
+pip-install: install-node-modules
+pip-install: install-scripts
+pip-install: update-bash-config
 
 .PHONY: uninstall
 uninstall:
@@ -110,7 +106,6 @@ uninstall:
 
 # Main targets end
 
-
 # Helper targets
 
 $(NNI_NODE_TARBALL):
@@ -144,6 +139,14 @@ install-python-modules:
 	#$(_INFO) Installing nnictl $(_END)
 	cd tools && $(PIP_INSTALL) $(PIP_MODE) .
 
+.PHONY: dev-install-python-modules
+dev-install-python-modules:
+	#$(_INFO) Installing Python SDK $(_END)
+	cd src/sdk/pynni && $(PIP_INSTALL) $(PIP_MODE) -e .
+	
+	#$(_INFO) Installing nnictl $(_END)
+	cd tools && $(PIP_INSTALL) $(PIP_MODE) -e .
+
 .PHONY: install-node-modules
 install-node-modules:
 	#$(_INFO) Installing NNI Package $(_END)
@@ -153,36 +156,16 @@ install-node-modules:
 	$(NNI_YARN) --prod --cwd $(NNI_PKG_FOLDER)
 	cp -r src/webui/build $(NNI_PKG_FOLDER)/static
 
-.PHONY: install-dev-modules
-install-dev-modules:
-	#$(_INFO) Installing Python SDK $(_END)
-	cd src/sdk/pynni && $(PIP_INSTALL) $(PIP_MODE) -e .
-	
-	#$(_INFO) Installing nnictl $(_END)
-	cd tools && $(PIP_INSTALL) $(PIP_MODE) -e .
-	
+.PHONY: dev-install-node-modules
+dev-install-node-modules:
 	#$(_INFO) Installing NNI Package $(_END)
 	rm -rf $(NNI_PKG_FOLDER)
 	ln -sf ${PWD}/src/nni_manager/dist $(NNI_PKG_FOLDER)
 	ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_PKG_FOLDER)/node_modules
 	ln -sf ${PWD}/src/webui/build $(NNI_PKG_FOLDER)/static
 
-
 .PHONY: install-scripts
 install-scripts:
-	mkdir -p $(BIN_FOLDER)
-
-	touch $(BIN_FOLDER)/nnictl
-	echo "#!/usr/bin/python3" >>$(BIN_FOLDER)/nnictl
-	echo "# -*- coding: utf-8 -*-" >>$(BIN_FOLDER)/nnictl
-	echo "import re" >>$(BIN_FOLDER)/nnictl
-	echo "import sys" >>$(BIN_FOLDER)/nnictl
-	echo "from nnicmd.nnictl import parse_args" >>$(BIN_FOLDER)/nnictl
-	echo "if __name__ == '__main__':" >>$(BIN_FOLDER)/nnictl
-	echo "    sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$$', '', sys.argv[0])" >>$(BIN_FOLDER)/nnictl
-	echo "    sys.exit(parse_args())" >>$(BIN_FOLDER)/nnictl
-
-	chmod +x $(BIN_FOLDER)/nnictl
 	install -Dm644 tools/bash-completion $(BASH_COMP_SCRIPT)
 
 .PHONY: update-bash-config
@@ -201,7 +184,6 @@ else
 update-bash-config: ;
 endif
 
-
 .PHONY: check-perm
 ifdef _ROOT
 check-perm:
@@ -213,13 +195,12 @@ else
 check-perm: ;
 endif
 
-
-.PHONY: check-dev-env
-check-dev-env:
-	#$(_INFO) Checking developing environment... $(_END)
+.PHONY: dev-check-perm
 ifdef _ROOT
+dev-check-perm:
 	$(error You should not develop NNI as root)
+else
+dev-check-perm: ;
 endif
-	#$(_INFO) Pass! $(_END)
 
 # Helper targets end

README.Makefile.md

@@ -5,9 +5,10 @@ NNI uses GNU make for building and installing.
 The `Makefile` offers standard targets `build`, `install`, and `uninstall`, as well as alternative installation targets for different setup:
 
 * `easy-install`: target for non-expert users, which handles everything automatically;
-* `pip-install`: target in favor of `setup.py`;
+* `dev-easy-install`: target for developer users, which handles everything automatically;
+* `install`: target for NNI normal users, which installs NNI by copying files;
 * `dev-install`: target for NNI contributors, which installs NNI as symlinks instead of copying files;
-* `remote-machine-install`: target that only installs core Python library for remote machine workers.
+* `pip-install`: target in favor of `setup.py`;
 
 The targets will be detailed later.
 
@@ -56,7 +57,6 @@ Available variables are listed below:
 |--------------------|---------------------------------------------------------|-----------------------------------|-------------------------------------------------|
 | `BIN_PATH`         | Path for executables                                    | `~/.local/bin`                    | `/usr/bin`                                      |
 | `INSTALL_PREFIX`   | Path for Node.js modules (a suffix `nni` will be added) | `~/.local`                        | `/usr/share`                                    |
-| `EXAMPLES_PATH`    | Path for NNI examples                                   | `~/nni/examples`                  | `$INSTALL_PREFIX/nni/examples`                  |
 | `BASH_COMP_SCRIPT` | Path of bash completion script                          | `~/.bash_completion.d/nnictl`     | `/usr/share/bash-completion/completions/nnictl` |
 | `PIP_MODE`         | Arguments for `python3 setup.py install`                | `--user` if `VIRTUAL_ENV` not set | (empty)                                         |
 | `NODE_PATH`        | Path to install Node.js runtime                         | `$INSTALL_PREFIX/nni/node`        | `$INSTALL_PREFIX/nni/node`                      |
@@ -71,19 +71,18 @@ If the path to copy files is different from where they will run (e.g. when creat
 
 The workflow of each installation targets is listed below:
 
-| Target                   | Workflow                                                             |
-|--------------------------|----------------------------------------------------------------------|
-| `install`                | Install Python packages, Node.js modules, NNI scripts, and examples  |
-| `easy-install`           | Install dependencies, build, install NNI, and edit `~/.bashrc`       |
-| `pip-install`            | Install dependencies, build, install NNI excluding Python packages   |
-| `dev-install`            | Install Python and Node.js modules as symlinks, then install scripts |
-| `remote-machine-install` | Install `nni` Python package                                         |
+| Target                   | Workflow                                                                   |
+|--------------------------|----------------------------------------------------------------------------|
+| `easy-install`           | Install dependencies, build, install NNI, and edit `~/.bashrc`             |
+| `dev-easy-install`       | Install dependencies, build, install NNI as symlinks, and edit `~/.bashrc` |
+| `install`                | Install Python packages, Node.js modules, NNI scripts, and examples        |
+| `dev-install`            | Install Python and Node.js modules as symlinks, then install scripts       |
+| `pip-install`            | Install dependencies, build, install NNI excluding Python packages         |
 
 ## TODO
 
 * `clean` target
 * `test` target
 * `lint` target
-* Exclude tuners and their dependencies from `remote-machine-install`
 * Test cases for each target
 * Review variables

README.md

@@ -24,7 +24,7 @@ The tool dispatches and runs trial jobs that generated by tuning algorithms to s
 **Install through pip** 
 * We only support Linux in current stage, Ubuntu 16.04 or higher are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
 ```bash
-    pip3 install --user nni-pkg
+    python3 -m pip install --user nni-pkg
 ```
 
 **Install through source code**

azure-pipelines.yml

@@ -6,7 +6,7 @@ strategy:
       PYTHON_VERSION: '3.6'
 
 steps:
-  - script: pip3 install --upgrade pip setuptools
+  - script: python3 -m pip install --upgrade pip setuptools
     displayName: 'Install python tools'
   - script: |
       source install.sh

deployment/docker/Dockerfile

@@ -47,23 +47,23 @@ RUN DEBIAN_FRONTEND=noninteractive && \
     rm -rf /var/lib/apt/lists/*
 
 # numpy 1.14.3  scipy 1.1.0 
-RUN pip3 --no-cache-dir install \
+RUN python3 -m pip --no-cache-dir install \
     numpy==1.14.3 scipy==1.1.0 
 
 #
 #Install NNI
 #
-RUN pip3 install --user nni-pkg
+RUN python3 -m pip install --user nni-pkg
 
 #
 #Tensorflow 1.10.0
 #
-RUN pip3 --no-cache-dir install tensorflow-gpu==1.10.0
+RUN python3 -m pip --no-cache-dir install tensorflow-gpu==1.10.0
 
 #
 #Keras 2.1.6
 #
-RUN pip3 --no-cache-dir install Keras==2.1.6
+RUN python3 -m pip --no-cache-dir install Keras==2.1.6
 
 ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin:
 

deployment/pypi/Makefile

@@ -2,7 +2,7 @@ CWD := $(PWD)/
 
 .PHONY: build
 build:
-	pip3 install --user --upgrade setuptools wheel
+	python3 -m pip install --user --upgrade setuptools wheel
 	wget https://aka.ms/nodejs-download -O $(CWD)node-linux-x64.tar.xz
 	rm -rf $(CWD)node-linux-x64
 	mkdir $(CWD)node-linux-x64
@@ -25,5 +25,5 @@ build:
 
 .PHONY: upload
 upload:
-	pip3 install --user --upgrade twine
-	twine upload dist/*
\ No newline at end of file
+	python3 -m pip install --user --upgrade twine
+	python3 -m twine upload dist/*
\ No newline at end of file

deployment/pypi/setup.py

@@ -22,7 +22,12 @@ setuptools.setup(
     packages = setuptools.find_packages(),
     python_requires = '>=3.5',
     install_requires = [
-        'nni'
+        'nni',
+        'schema',
+        'pyyaml',
+        'psutil',
+        'requests',
+        'paramiko'
     ],
     classifiers = [
         'Programming Language :: Python :: 3',

docs/GetStarted.md

@@ -8,13 +8,13 @@
       git
       wget
 
-    python pip should also be correctly installed. You could use "pip3 -v" to check in Linux.
+    python pip should also be correctly installed. You could use "python3 -m pip -v" to check in Linux.
     
     * Note: we don't support virtual environment in current releases.
 
 * __Install NNI through pip__
 
-      pip3 install --user nni-pkg
+      python3 -m pip install --user nni-pkg
 
 * __Install NNI through source code__
    

docs/InstallNNI_Ubuntu.md

@@ -14,7 +14,7 @@
 
 * __Install NNI through pip__
 
-      pip3 install --user nni-pkg
+      python3 -m pip install --user nni-pkg
 
 * __Install NNI through source code__
    

docs/RemoteMachineMode.md

@@ -15,7 +15,7 @@ For remote machines that are used only to run trials but not the nnictl, you can
 
 * __Install python SDK through pip__
 
-      pip3 install --user nni
+      python3 -m pip install --user nni
 
 * __Install python SDK through source code__
 

docs/tutorial_2_RemoteMachineMode.md

@@ -15,7 +15,7 @@ For remote machines that are used only to run trials but not the nnictl, you can
 
 * __Install python SDK through pip__
 
-      pip3 install --user nni
+      python3 -m pip install --user nni
 
 * __Install python SDK through source code__
 

examples/trials/pytorch_cifar10/requirement.txt

@@ -2,5 +2,5 @@ This example requires pytorch.
 pytorch install package should be chosen based on python version and cuda version.
 
 Here is an example of the environment python==3.5 and cuda == 8.0, then using the following commands to install pytorch:
-pip3 install http://download.pytorch.org/whl/cu80/torch-0.4.1-cp35-cp35m-linux_x86_64.whl 
-pip3 install torchvision
\ No newline at end of file
+python3 -m pip install http://download.pytorch.org/whl/cu80/torch-0.4.1-cp35-cp35m-linux_x86_64.whl 
+python3 -m pip install torchvision
\ No newline at end of file

src/nni_manager/training_service/pai/paiData.ts

@@ -56,7 +56,7 @@ if python3 -c 'import nni' > /dev/null 2>&1; then
   return
 else
   # Install nni
-  pip3 install -v --user nni-pkg
+  python3 -m pip install --user nni-pkg
 fi`;
 
 export const PAI_TRIAL_COMMAND_FORMAT: string =

tools/README.md

@@ -14,11 +14,11 @@ python >= 3.5
  2. Use pip3 to install packages
     * Install for current user:
 
-          pip3 install --user -e .
+          python3 -m pip install --user -e .
 
     * Install for all users:
 
-          pip3 install -e .
+          python3 -m pip install -e .
 
  
 1. Change the mode of nnictl file 

tools/nnicmd/launcher.py

@@ -35,8 +35,9 @@ from .constants import *
 import time
 import random
 import string
-from os import path
 import site
+from pathlib import Path
+
 
 def get_log_path(config_file_name):
     '''generate stdout and stderr log path'''
@@ -66,7 +67,12 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None
         exit(1)
 
     print_normal('Starting restful server...')
-    cmds = ['node', path.join(site.USER_BASE, 'nni_pkg', 'main.js'), '--port', str(port), '--mode', platform, '--start_mode', mode]
+    if os.geteuid() == 0:
+        site_dir = site.getsitepackages()[0]
+    else:
+        site_dir = site.getusersitepackages()
+    python_dir = str(Path(site_dir).parents[2])
+    cmds = ['node', os.path.join(python_dir, 'nni_pkg', 'main.js'), '--port', str(port), '--mode', platform, '--start_mode', mode]
     if mode == 'resume':
         cmds += ['--experiment_id', experiment_id]
     stdout_full_path, stderr_full_path = get_log_path(config_file_name)
@@ -77,7 +83,7 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None
     log_header = LOG_HEADER % str(time_now)
     stdout_file.write(log_header)
     stderr_file.write(log_header)
-    process = Popen(cmds, cwd=path.join(site.USER_BASE, 'nni_pkg'), stdout=stdout_file, stderr=stderr_file)
+    process = Popen(cmds, cwd=os.path.join(python_dir, 'nni_pkg'), stdout=stdout_file, stderr=stderr_file)
     return process, str(time_now)
 
 def set_trial_config(experiment_config, port, config_file_name):

tools/nnicmd/package_management.py

@@ -29,7 +29,7 @@ def process_install(package_name):
         print_error('{0} is not supported!' % package_name)
     else:
         requirements_path = os.path.join(nni.__path__[0], PACKAGE_REQUIREMENTS[package_name])
-        cmds = 'cd ' + requirements_path + ' && pip3 install --user -r requirements.txt'
+        cmds = 'cd ' + requirements_path + ' && python3 -m pip install --user -r requirements.txt'
         call(cmds, shell=True)
 
 def package_install(args):

tools/setup.py

@@ -20,4 +20,9 @@ setuptools.setup(
     description = 'NNI control for Neural Network Intelligence project',
     license = 'MIT',
     url = 'https://github.com/Microsoft/nni',
+    entry_points = {
+        'console_scripts' : [
+            'nnictl = nnicmd.nnictl:parse_args'
+        ]
+    }
 )