To install the whole nni in an virtual environment (#538)

* support venv

* adapt venv

* adapt venv

* adapt venv

* adapt venv

* new test

* new test

* new test

* support venv

* support venv

* support venv

* support venv

* support venv

* support venv

* support venv

* colorful output for mac

* colorful output for mac

* permission denied in /tmp

* permission denied in /tmp

* permission denied in /tmp

* remove unused variable

* final

* remove build python

Makefile

 # Setting variables
-
+SHELL := /bin/bash
 PIP_INSTALL := python3 -m pip install --no-cache-dir
 PIP_UNINSTALL := python3 -m pip uninstall
 
 ## Colorful output
-_INFO := $(shell echo -e '\e[1;36m')
-_WARNING := $(shell echo -e '\e[1;33m')
-_END := $(shell echo -e '\e[0m')
+_INFO := $(shell echo -e '\033[1;36m')
+_WARNING := $(shell echo -e '\033[1;33m')
+_END := $(shell echo -e '\033[0m')
 
 ## Detect OS
 UNAME_S := $(shell uname -s)
@@ -19,15 +19,21 @@ else
 endif
 
 ## Install directories
+ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])')
+IS_SYS_PYTHON ?= $(shell [[ $(ROOT_FOLDER) == /usr* || $(ROOT_FOLDER) == /Library* ]] && echo TRUE || echo FALSE)
+
 ifeq ($(shell id -u), 0)  # is root
     _ROOT := 1
-    ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])')
     BASH_COMP_PREFIX ?= /usr/share/bash-completion/completions
 else  # is normal user
-    ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])')
+    ifeq (TRUE, $(IS_SYS_PYTHON))
+        ROOT_FOLDER := $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])')
+    endif
     ifndef VIRTUAL_ENV
+        ifeq (, $(shell echo $$PATH | grep 'conda'))
             PIP_MODE ?= --user
         endif
+    endif
     BASH_COMP_PREFIX ?= ${HOME}/.bash_completion.d
 endif
 BASH_COMP_SCRIPT := $(BASH_COMP_PREFIX)/nnictl
@@ -38,11 +44,13 @@ BIN_FOLDER ?= $(ROOT_FOLDER)/bin
 NNI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni
 
 ## Dependency information
-NNI_NODE_TARBALL ?= /tmp/nni-node-$(OS_SPEC)-x64.tar.xz
-NNI_NODE_FOLDER = /tmp/nni-node-$(OS_SPEC)-x64
+NNI_DEPENDENCY_FOLDER = /tmp/$(USER)
+$(shell mkdir -p $(NNI_DEPENDENCY_FOLDER))
+NNI_NODE_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64.tar.xz
+NNI_NODE_FOLDER = $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64
 NNI_NODE ?= $(BIN_FOLDER)/node
-NNI_YARN_TARBALL ?= /tmp/nni-yarn.tar.gz
-NNI_YARN_FOLDER ?= /tmp/nni-yarn
+NNI_YARN_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn.tar.gz
+NNI_YARN_FOLDER ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn
 NNI_YARN := PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn
 
 ## Version number
@@ -57,10 +65,6 @@ build:
 	cd src/nni_manager && $(NNI_YARN) && $(NNI_YARN) build
 	#$(_INFO) Building WebUI $(_END)
 	cd src/webui && $(NNI_YARN) && $(NNI_YARN) build
-	#$(_INFO) Building Python SDK $(_END)
-	cd src/sdk/pynni && python3 setup.py build
-	#$(_INFO) Building nnictl $(_END)
-	cd tools && python3 setup.py build
 
 # All-in-one target for non-expert users
 # Installs NNI as well as its dependencies, and update bashrc to set PATH

README.md

@@ -36,7 +36,7 @@ We encourage researchers and students leverage these projects to accelerate the
 **Install through pip** 	
 * We support Linux and MacOS in current stage, Ubuntu 16.04 or higher, along with MacOS 10.14.1 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.	
 ```bash	
-    python3 -m pip install --user --upgrade nni
+    python3 -m pip install --upgrade nni
 ```
 * Note:
   * If you are in docker container (as root), please remove `--user` from the installation command.

install.sh

 #!/bin/bash
 make easy-install
-source ~/.bashrc

tools/nni_cmd/launcher.py

@@ -32,9 +32,9 @@ from .url_utils import cluster_metadata_url, experiment_url, get_local_urls
 from .config_utils import Config, Experiments
 from .common_utils import get_yml_content, get_json_content, print_error, print_normal, print_warning, detect_process, detect_port
 from .constants import *
-import time
 import random
 import site
+import time
 from pathlib import Path
 
 def get_log_path(config_file_name):
@@ -72,16 +72,26 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None
         exit(1)
 
     print_normal('Starting restful server...')
-    python_dir = str(Path(site.getusersitepackages()).parents[2])
-    entry_file = os.path.join(python_dir, 'nni', 'main.js')
-    entry_dir = os.path.join(python_dir, 'nni')
-    local_entry_dir = entry_dir
-    if not os.path.isfile(entry_file):
-        python_dir = str(Path(site.getsitepackages()[0]).parents[2])
+    # Find nni lib from the following locations in order
+    sys_wide_python = True
+    python_sitepackage = site.getsitepackages()[0]
+    # If system-wide python is used, we will give priority to using user-sitepackage given that nni exists there
+    if python_sitepackage.startswith('/usr') or python_sitepackage.startswith('/Library'):
+        local_python_dir = str(Path(site.getusersitepackages()).parents[2])
+        entry_file = os.path.join(local_python_dir, 'nni', 'main.js')
+        entry_dir = os.path.join(local_python_dir, 'nni')
+    else:
+        # If this python is not system-wide python, we will use its site-package directly
+        sys_wide_python = False
+
+    if not sys_wide_python or not os.path.isfile(entry_file):
+        python_dir = str(Path(python_sitepackage).parents[2])
         entry_file = os.path.join(python_dir, 'nni', 'main.js')
         entry_dir = os.path.join(python_dir, 'nni')
+        # Nothing is found
         if not os.path.isfile(entry_file):
-            raise Exception('Fail to find main.js under both %s and %s!' % (local_entry_dir, entry_dir))
+            raise Exception('Fail to find nni under both "%s" and "%s"' % (local_python_dir, python_dir))
+
     cmds = ['node', entry_file, '--port', str(port), '--mode', platform, '--start_mode', mode]
     if mode == 'resume':
         cmds += ['--experiment_id', experiment_id]