Commit 1159c855 authored by Zejun Lin's avatar Zejun Lin Committed by QuanluZhang
Browse files

To install the whole nni in an virtual environment (#538)

* support venv

* adapt venv

* adapt venv

* adapt venv

* adapt venv

* new test

* new test

* new test

* support venv

* support venv

* support venv

* support venv

* support venv

* support venv

* support venv

* colorful output for mac

* colorful output for mac

* permission denied in /tmp

* permission denied in /tmp

* permission denied in /tmp

* remove unused variable

* final

* remove build python
parent e3332641
# Setting variables # Setting variables
SHELL := /bin/bash
PIP_INSTALL := python3 -m pip install --no-cache-dir PIP_INSTALL := python3 -m pip install --no-cache-dir
PIP_UNINSTALL := python3 -m pip uninstall PIP_UNINSTALL := python3 -m pip uninstall
## Colorful output ## Colorful output
_INFO := $(shell echo -e '\e[1;36m') _INFO := $(shell echo -e '\033[1;36m')
_WARNING := $(shell echo -e '\e[1;33m') _WARNING := $(shell echo -e '\033[1;33m')
_END := $(shell echo -e '\e[0m') _END := $(shell echo -e '\033[0m')
## Detect OS ## Detect OS
UNAME_S := $(shell uname -s) UNAME_S := $(shell uname -s)
...@@ -19,15 +19,21 @@ else ...@@ -19,15 +19,21 @@ else
endif endif
## Install directories ## Install directories
ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])')
IS_SYS_PYTHON ?= $(shell [[ $(ROOT_FOLDER) == /usr* || $(ROOT_FOLDER) == /Library* ]] && echo TRUE || echo FALSE)
ifeq ($(shell id -u), 0) # is root ifeq ($(shell id -u), 0) # is root
_ROOT := 1 _ROOT := 1
ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]).parents[2])')
BASH_COMP_PREFIX ?= /usr/share/bash-completion/completions BASH_COMP_PREFIX ?= /usr/share/bash-completion/completions
else # is normal user else # is normal user
ROOT_FOLDER ?= $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])') ifeq (TRUE, $(IS_SYS_PYTHON))
ROOT_FOLDER := $(shell python3 -c 'import site; from pathlib import Path; print(Path(site.getusersitepackages()).parents[2])')
endif
ifndef VIRTUAL_ENV ifndef VIRTUAL_ENV
ifeq (, $(shell echo $$PATH | grep 'conda'))
PIP_MODE ?= --user PIP_MODE ?= --user
endif endif
endif
BASH_COMP_PREFIX ?= ${HOME}/.bash_completion.d BASH_COMP_PREFIX ?= ${HOME}/.bash_completion.d
endif endif
BASH_COMP_SCRIPT := $(BASH_COMP_PREFIX)/nnictl BASH_COMP_SCRIPT := $(BASH_COMP_PREFIX)/nnictl
...@@ -38,11 +44,13 @@ BIN_FOLDER ?= $(ROOT_FOLDER)/bin ...@@ -38,11 +44,13 @@ BIN_FOLDER ?= $(ROOT_FOLDER)/bin
NNI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni NNI_PKG_FOLDER ?= $(ROOT_FOLDER)/nni
## Dependency information ## Dependency information
NNI_NODE_TARBALL ?= /tmp/nni-node-$(OS_SPEC)-x64.tar.xz NNI_DEPENDENCY_FOLDER = /tmp/$(USER)
NNI_NODE_FOLDER = /tmp/nni-node-$(OS_SPEC)-x64 $(shell mkdir -p $(NNI_DEPENDENCY_FOLDER))
NNI_NODE_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64.tar.xz
NNI_NODE_FOLDER = $(NNI_DEPENDENCY_FOLDER)/nni-node-$(OS_SPEC)-x64
NNI_NODE ?= $(BIN_FOLDER)/node NNI_NODE ?= $(BIN_FOLDER)/node
NNI_YARN_TARBALL ?= /tmp/nni-yarn.tar.gz NNI_YARN_TARBALL ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn.tar.gz
NNI_YARN_FOLDER ?= /tmp/nni-yarn NNI_YARN_FOLDER ?= $(NNI_DEPENDENCY_FOLDER)/nni-yarn
NNI_YARN := PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn NNI_YARN := PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn
## Version number ## Version number
...@@ -57,10 +65,6 @@ build: ...@@ -57,10 +65,6 @@ build:
cd src/nni_manager && $(NNI_YARN) && $(NNI_YARN) build cd src/nni_manager && $(NNI_YARN) && $(NNI_YARN) build
#$(_INFO) Building WebUI $(_END) #$(_INFO) Building WebUI $(_END)
cd src/webui && $(NNI_YARN) && $(NNI_YARN) build cd src/webui && $(NNI_YARN) && $(NNI_YARN) build
#$(_INFO) Building Python SDK $(_END)
cd src/sdk/pynni && python3 setup.py build
#$(_INFO) Building nnictl $(_END)
cd tools && python3 setup.py build
# All-in-one target for non-expert users # All-in-one target for non-expert users
# Installs NNI as well as its dependencies, and update bashrc to set PATH # Installs NNI as well as its dependencies, and update bashrc to set PATH
......
...@@ -36,7 +36,7 @@ We encourage researchers and students leverage these projects to accelerate the ...@@ -36,7 +36,7 @@ We encourage researchers and students leverage these projects to accelerate the
**Install through pip** **Install through pip**
* We support Linux and MacOS in current stage, Ubuntu 16.04 or higher, along with MacOS 10.14.1 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`. * We support Linux and MacOS in current stage, Ubuntu 16.04 or higher, along with MacOS 10.14.1 are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
```bash ```bash
python3 -m pip install --user --upgrade nni python3 -m pip install --upgrade nni
``` ```
* Note: * Note:
* If you are in docker container (as root), please remove `--user` from the installation command. * If you are in docker container (as root), please remove `--user` from the installation command.
......
#!/bin/bash #!/bin/bash
make easy-install make easy-install
source ~/.bashrc
...@@ -32,9 +32,9 @@ from .url_utils import cluster_metadata_url, experiment_url, get_local_urls ...@@ -32,9 +32,9 @@ from .url_utils import cluster_metadata_url, experiment_url, get_local_urls
from .config_utils import Config, Experiments from .config_utils import Config, Experiments
from .common_utils import get_yml_content, get_json_content, print_error, print_normal, print_warning, detect_process, detect_port from .common_utils import get_yml_content, get_json_content, print_error, print_normal, print_warning, detect_process, detect_port
from .constants import * from .constants import *
import time
import random import random
import site import site
import time
from pathlib import Path from pathlib import Path
def get_log_path(config_file_name): def get_log_path(config_file_name):
...@@ -72,16 +72,26 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None ...@@ -72,16 +72,26 @@ def start_rest_server(port, platform, mode, config_file_name, experiment_id=None
exit(1) exit(1)
print_normal('Starting restful server...') print_normal('Starting restful server...')
python_dir = str(Path(site.getusersitepackages()).parents[2]) # Find nni lib from the following locations in order
entry_file = os.path.join(python_dir, 'nni', 'main.js') sys_wide_python = True
entry_dir = os.path.join(python_dir, 'nni') python_sitepackage = site.getsitepackages()[0]
local_entry_dir = entry_dir # If system-wide python is used, we will give priority to using user-sitepackage given that nni exists there
if not os.path.isfile(entry_file): if python_sitepackage.startswith('/usr') or python_sitepackage.startswith('/Library'):
python_dir = str(Path(site.getsitepackages()[0]).parents[2]) local_python_dir = str(Path(site.getusersitepackages()).parents[2])
entry_file = os.path.join(local_python_dir, 'nni', 'main.js')
entry_dir = os.path.join(local_python_dir, 'nni')
else:
# If this python is not system-wide python, we will use its site-package directly
sys_wide_python = False
if not sys_wide_python or not os.path.isfile(entry_file):
python_dir = str(Path(python_sitepackage).parents[2])
entry_file = os.path.join(python_dir, 'nni', 'main.js') entry_file = os.path.join(python_dir, 'nni', 'main.js')
entry_dir = os.path.join(python_dir, 'nni') entry_dir = os.path.join(python_dir, 'nni')
# Nothing is found
if not os.path.isfile(entry_file): if not os.path.isfile(entry_file):
raise Exception('Fail to find main.js under both %s and %s!' % (local_entry_dir, entry_dir)) raise Exception('Fail to find nni under both "%s" and "%s"' % (local_python_dir, python_dir))
cmds = ['node', entry_file, '--port', str(port), '--mode', platform, '--start_mode', mode] cmds = ['node', entry_file, '--port', str(port), '--mode', platform, '--start_mode', mode]
if mode == 'resume': if mode == 'resume':
cmds += ['--experiment_id', experiment_id] cmds += ['--experiment_id', experiment_id]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment