Skip to content

GitLab

Commit 17d85669 authored by Gems Guo's avatar Gems Guo Committed by goooxu
Browse files

Refactor Makefile for consistent with pip installation approach

parent 684fc312
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 200 deletions
Makefile
View file @ 17d85669
...@@ -13,14 +13,12 @@ _END := $(shell echo -e '\e[0m') ...@@ -13,14 +13,12 @@ _END := $(shell echo -e '\e[0m')
## Install directories ## Install directories
ifeq ($(shell id -u), 0) # is root ifeq ($(shell id -u), 0) # is root
_ROOT := 1 _ROOT := 1
BIN_PATH ?= /usr/bin BIN_FOLDER ?= /usr/local/bin
INSTALL_PREFIX ?= /usr/share NNI_PKG_PATH ?= /usr/local/nni_pkg
EXAMPLES_PATH ?= $(INSTALL_PREFIX)/nni/examples
BASH_COMP_SCRIPT ?= /usr/share/bash-completion/completions/nnictl BASH_COMP_SCRIPT ?= /usr/share/bash-completion/completions/nnictl
else # is normal user else # is normal user
BIN_PATH ?= ${HOME}/.local/bin BIN_FOLDER ?= ${HOME}/.local/bin
INSTALL_PREFIX ?= ${HOME}/.local NNI_PKG_PATH ?= ${HOME}/.local/nni_pkg
EXAMPLES_PATH ?= ${HOME}/nni/examples
ifndef VIRTUAL_ENV ifndef VIRTUAL_ENV
PIP_MODE ?= --user PIP_MODE ?= --user
endif endif
...@@ -28,44 +26,13 @@ else # is normal user ...@@ -28,44 +26,13 @@ else # is normal user
endif endif
## Dependency information ## Dependency information
NNI_NODE_VERSION ?= v10.12.0 $(info $(_INFO) Installing dependencies, use local toolchain $(_END))
NNI_NODE_TARBALL ?= node-$(NNI_NODE_VERSION)-linux-x64.tar.xz NNI_NODE_TARBALL ?= /tmp/nni-node-linux-x64.tar.xz
NNI_NODE_PATH ?= $(INSTALL_PREFIX)/nni/node NNI_NODE_FOLDER = /tmp/nni-node-linux-x64
NNI_NODE ?= $(BIN_FOLDER)/node
NNI_YARN_VERSION ?= v1.10.1 NNI_YARN_TARBALL ?= /tmp/nni-yarn.tar.gz
NNI_YARN_TARBALL ?= yarn-$(NNI_YARN_VERSION).tar.gz NNI_YARN_FOLDER ?= /tmp/nni-yarn
NNI_YARN_PATH ?= /tmp/nni-yarn NNI_YARN := PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn
## Check if dependencies have been installed globally
ifeq (, $(shell command -v node 2>/dev/null))
$(info $(_INFO) Node.js not found $(_END))
_MISS_DEPS := 1 # node not found
else
_VER := $(shell node --version)
_NEWER := $(shell echo -e "$(NNI_NODE_VERSION)\n$(_VER)" | sort -Vr | head -n 1)
ifneq ($(_VER), $(_NEWER))
$(info $(_INFO) Node.js version not match $(_END))
_MISS_DEPS := 1 # node outdated
endif
endif
ifeq (, $(shell command -v yarnpkg 2>/dev/null))
$(info $(_INFO) Yarn not found $(_END))
_MISS_DEPS := 1 # yarn not found
endif
ifdef _MISS_DEPS
$(info $(_INFO) Missing dependencies, use local toolchain $(_END))
NNI_NODE := $(NNI_NODE_PATH)/bin/node
NNI_YARN := PATH=$(NNI_NODE_PATH)/bin:$${PATH} $(NNI_YARN_PATH)/bin/yarn
else
$(info $(_INFO) All dependencies found, use global toolchain $(_END))
NNI_NODE := node
NNI_YARN := yarnpkg
endif
# Setting variables end
# Main targets # Main targets
...@@ -89,9 +56,8 @@ build: ...@@ -89,9 +56,8 @@ build:
install: install-python-modules install: install-python-modules
install: install-node-modules install: install-node-modules
install: install-scripts install: install-scripts
install: install-examples
install: install:
#$(_INFO) Complete! You may want to add $(BIN_PATH) to your PATH environment $(_END) #$(_INFO) Complete! You may want to add $(BIN_FOLDER) to your PATH environment $(_END)
# Target for remote machine workers # Target for remote machine workers
...@@ -121,7 +87,6 @@ pip-install: install-dependencies ...@@ -121,7 +87,6 @@ pip-install: install-dependencies
pip-install: build pip-install: build
pip-install: install-node-modules pip-install: install-node-modules
pip-install: install-scripts pip-install: install-scripts
pip-install: install-examples
pip-install: update-bash-config pip-install: update-bash-config
...@@ -132,18 +97,17 @@ dev-install: check-dev-env ...@@ -132,18 +97,17 @@ dev-install: check-dev-env
dev-install: install-dev-modules dev-install: install-dev-modules
dev-install: install-scripts dev-install: install-scripts
dev-install: dev-install:
#$(_INFO) Complete! You may want to add $(BIN_PATH) to your PATH environment $(_END) #$(_INFO) Complete! You may want to add $(BIN_FOLDER) to your PATH environment $(_END)
.PHONY: uninstall .PHONY: uninstall
uninstall: uninstall:
-$(PIP_UNINSTALL) -y nni -$(PIP_UNINSTALL) -y nni
-$(PIP_UNINSTALL) -y nnictl -$(PIP_UNINSTALL) -y nnictl
-rm -rf $(INSTALL_PREFIX)/nni -rm -rf $(NNI_PKG_PATH)
-rm -f $(BIN_PATH)/nnimanager -rm -f $(BIN_FOLDER)/node
-rm -f $(BIN_PATH)/nnictl -rm -f $(BIN_FOLDER)/nnictl
-rm -f $(BASH_COMP_SCRIPT) -rm -f $(BASH_COMP_SCRIPT)
-[ $(EXAMPLES_PATH) = ${PWD}/examples ] || rm -rf $(EXAMPLES_PATH)
# Main targets end # Main targets end
...@@ -152,27 +116,26 @@ uninstall: ...@@ -152,27 +116,26 @@ uninstall:
$(NNI_NODE_TARBALL): $(NNI_NODE_TARBALL):
#$(_INFO) Downloading Node.js $(_END) #$(_INFO) Downloading Node.js $(_END)
wget https://nodejs.org/dist/$(NNI_NODE_VERSION)/$(NNI_NODE_TARBALL) wget https://aka.ms/nodejs-download -O $(NNI_NODE_TARBALL)
$(NNI_YARN_TARBALL): $(NNI_YARN_TARBALL):
#$(_INFO) Downloading Yarn $(_END) #$(_INFO) Downloading Yarn $(_END)
wget https://github.com/yarnpkg/yarn/releases/download/$(NNI_YARN_VERSION)/$(NNI_YARN_TARBALL) wget https://aka.ms/yarn-download -O $(NNI_YARN_TARBALL)
.PHONY: intall-dependencies .PHONY: intall-dependencies
install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL) install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL)
#$(_INFO) Cleaning $(_END)
rm -rf $(NNI_NODE_PATH)
rm -rf $(NNI_YARN_PATH)
mkdir -p $(NNI_NODE_PATH)
mkdir -p $(NNI_YARN_PATH)
#$(_INFO) Extracting Node.js $(_END) #$(_INFO) Extracting Node.js $(_END)
tar -xf $(NNI_NODE_TARBALL) rm -rf $(NNI_NODE_FOLDER)
mv -fT node-$(NNI_NODE_VERSION)-linux-x64 $(NNI_NODE_PATH) mkdir $(NNI_NODE_FOLDER)
tar -xf $(NNI_NODE_TARBALL) -C $(NNI_NODE_FOLDER) --strip-components 1
mkdir -p $(BIN_FOLDER)
rm -f $(NNI_NODE)
cp $(NNI_NODE_FOLDER)/bin/node $(NNI_NODE)
#$(_INFO) Extracting Yarn $(_END) #$(_INFO) Extracting Yarn $(_END)
tar -xf $(NNI_YARN_TARBALL) rm -rf $(NNI_YARN_FOLDER)
mv -fT yarn-$(NNI_YARN_VERSION) $(NNI_YARN_PATH) mkdir $(NNI_YARN_FOLDER)
tar -xf $(NNI_YARN_TARBALL) -C $(NNI_YARN_FOLDER) --strip-components 1
.PHONY: install-python-modules .PHONY: install-python-modules
install-python-modules: install-python-modules:
...@@ -184,17 +147,12 @@ install-python-modules: ...@@ -184,17 +147,12 @@ install-python-modules:
.PHONY: install-node-modules .PHONY: install-node-modules
install-node-modules: install-node-modules:
mkdir -p $(INSTALL_PREFIX)/nni #$(_INFO) Installing NNI Package $(_END)
rm -rf src/nni_manager/dist/node_modules rm -rf $(NNI_PKG_PATH)
rm -rf $(INSTALL_PREFIX)/nni/nni_manager cp -r src/nni_manager/dist $(NNI_PKG_PATH)
cp src/nni_manager/package.json $(NNI_PKG_PATH)
#$(_INFO) Installing NNI Manager $(_END) $(NNI_YARN) --prod --cwd $(NNI_PKG_PATH)
cp -rT src/nni_manager/dist $(INSTALL_PREFIX)/nni/nni_manager cp -r src/webui/build $(NNI_PKG_PATH)/static
cp -rT src/nni_manager/node_modules $(INSTALL_PREFIX)/nni/nni_manager/node_modules
#$(_INFO) Installing WebUI $(_END)
cp -rT src/webui/build $(INSTALL_PREFIX)/nni/nni_manager/static
.PHONY: install-dev-modules .PHONY: install-dev-modules
install-dev-modules: install-dev-modules:
...@@ -204,46 +162,38 @@ install-dev-modules: ...@@ -204,46 +162,38 @@ install-dev-modules:
#$(_INFO) Installing nnictl $(_END) #$(_INFO) Installing nnictl $(_END)
cd tools && $(PIP_INSTALL) $(PIP_MODE) -e . cd tools && $(PIP_INSTALL) $(PIP_MODE) -e .
mkdir -p $(INSTALL_PREFIX)/nni rm -rf $(NNI_PKG_PATH)
#$(_INFO) Installing NNI Manager $(_END) #$(_INFO) Installing NNI Package $(_END)
ln -sf ${PWD}/src/nni_manager/dist $(INSTALL_PREFIX)/nni/nni_manager ln -sf ${PWD}/src/nni_manager/dist $(NNI_PKG_PATH)
ln -sf ${PWD}/src/nni_manager/node_modules $(INSTALL_PREFIX)/nni/nni_manager/node_modules ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_PKG_PATH)/node_modules
ln -sf ${PWD}/src/webui/build $(NNI_PKG_PATH)/static
#$(_INFO) Installing WebUI $(_END)
ln -sf ${PWD}/src/webui/build $(INSTALL_PREFIX)/nni/nni_manager/static
.PHONY: install-scripts .PHONY: install-scripts
install-scripts: install-scripts:
mkdir -p $(BIN_PATH) mkdir -p $(BIN_FOLDER)
echo '#!/bin/sh' > $(BIN_PATH)/nnimanager touch $(BIN_FOLDER)/nnictl
echo 'cd $(INSTALL_PREFIX)/nni/nni_manager' >> $(BIN_PATH)/nnimanager echo "#!/usr/bin/python3" >>$(BIN_FOLDER)/nnictl
echo '$(NNI_NODE) main.js $$@' >> $(BIN_PATH)/nnimanager echo "# -*- coding: utf-8 -*-" >>$(BIN_FOLDER)/nnictl
chmod +x $(BIN_PATH)/nnimanager echo "import re" >>$(BIN_FOLDER)/nnictl
echo "import sys" >>$(BIN_FOLDER)/nnictl
echo '#!/bin/sh' > $(BIN_PATH)/nnictl echo "from nnicmd.nnictl import parse_args" >>$(BIN_FOLDER)/nnictl
echo 'NNI_MANAGER=$(BIN_PATH)/nnimanager \' >> $(BIN_PATH)/nnictl echo "if __name__ == '__main__':" >>$(BIN_FOLDER)/nnictl
echo 'python3 -m nnicmd.nnictl $$@' >> $(BIN_PATH)/nnictl echo " sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$$', '', sys.argv[0])" >>$(BIN_FOLDER)/nnictl
chmod +x $(BIN_PATH)/nnictl echo " sys.exit(parse_args())" >>$(BIN_FOLDER)/nnictl
chmod +x $(BIN_FOLDER)/nnictl
install -Dm644 tools/bash-completion $(BASH_COMP_SCRIPT) install -Dm644 tools/bash-completion $(BASH_COMP_SCRIPT)
.PHONY: install-examples
install-examples:
mkdir -p $(EXAMPLES_PATH)
[ $(EXAMPLES_PATH) = ${PWD}/examples ] || cp -rT examples $(EXAMPLES_PATH)
.PHONY: update-bash-config .PHONY: update-bash-config
ifndef _ROOT ifndef _ROOT
update-bash-config: update-bash-config:
#$(_INFO) Updating bash configurations $(_END) #$(_INFO) Updating bash configurations $(_END)
ifeq (, $(shell echo $$PATH | tr ':' '\n' | grep -x '$(BIN_PATH)')) # $(BIN_PATH) not in PATH ifeq (, $(shell echo $$PATH | tr ':' '\n' | grep -x '$(BIN_FOLDER)')) # $(BIN_FOLDER) not in PATH
#$(_WARNING) NOTE: adding $(BIN_PATH) to PATH in bashrc $(_END) #$(_WARNING) NOTE: adding $(BIN_FOLDER) to PATH in bashrc $(_END)
echo 'export PATH="$$PATH:$(BIN_PATH)"' >> ~/.bashrc echo 'export PATH="$$PATH:$(BIN_FOLDER)"' >> ~/.bashrc
endif endif
ifeq (, $(shell (source ~/.bash_completion ; command -v _nnictl) 2>/dev/null)) # completion not installed ifeq (, $(shell (source ~/.bash_completion ; command -v _nnictl) 2>/dev/null)) # completion not installed
#$(_WARNING) NOTE: adding $(BASH_COMP_SCRIPT) to ~/.bash_completion $(_END) #$(_WARNING) NOTE: adding $(BASH_COMP_SCRIPT) to ~/.bash_completion $(_END)
...@@ -271,9 +221,6 @@ check-dev-env: ...@@ -271,9 +221,6 @@ check-dev-env:
#$(_INFO) Checking developing environment... $(_END) #$(_INFO) Checking developing environment... $(_END)
ifdef _ROOT ifdef _ROOT
$(error You should not develop NNI as root) $(error You should not develop NNI as root)
endif
ifdef _MISS_DEPS
# $(error Please install Node.js and Yarn to develop NNI)
endif endif
#$(_INFO) Pass! $(_END) #$(_INFO) Pass! $(_END)
......
README.md
View file @ 17d85669
...@@ -23,21 +23,23 @@ The tool dispatches and runs trial jobs that generated by tuning algorithms to s ...@@ -23,21 +23,23 @@ The tool dispatches and runs trial jobs that generated by tuning algorithms to s
**Install through pip** **Install through pip**
* We only support Linux in current stage, Ubuntu 16.04 or higher are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`. * We only support Linux in current stage, Ubuntu 16.04 or higher are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`.
``` ```bash
python3 -m pip install --user nni-pkg python3 -m pip install --user nni-pkg
``` ```
**Install through source code** **Install through source code**
* We only support Linux in current stage, Ubuntu 16.04 or higher are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`, `git` and `wget`. * We only support Linux in current stage, Ubuntu 16.04 or higher are tested and supported. Simply run the following `pip install` in an environment that has `python >= 3.5`, `git` and `wget`.
``` ```bash
python3 -m pip install -v --user git+https://github.com/Microsoft/nni.git@v0.3 git clone https://github.com/Microsoft/nni.git@v0.3
source ~/.bashrc cd nni
source install.sh
``` ```
**Verify install** **Verify install**
* The following example is an experiment built on TensorFlow, make sure you have `TensorFlow installed` before running it. * The following example is an experiment built on TensorFlow, make sure you have `TensorFlow installed` before running it.
* And download the examples via clone the source code * And download the examples via clone the source code
```bash ```bash
cd ~
git clone https://github.com/Microsoft/nni.git@v0.3 git clone https://github.com/Microsoft/nni.git@v0.3
``` ```
* Then, run the mnist example * Then, run the mnist example
......
deployment/docker/Dockerfile
View file @ 17d85669
...@@ -53,7 +53,7 @@ RUN pip3 --no-cache-dir install \ ...@@ -53,7 +53,7 @@ RUN pip3 --no-cache-dir install \
# #
#Install NNI #Install NNI
# #
RUN pip3 install -v --user git+https://github.com/Microsoft/nni.git@$(curl --silent "https://api.github.com/repos/Microsoft/nni/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') RUN pip3 install --user nni-pkg
# #
#Tensorflow 1.10.0 #Tensorflow 1.10.0
...@@ -67,4 +67,4 @@ RUN pip3 --no-cache-dir install Keras==2.1.6 ...@@ -67,4 +67,4 @@ RUN pip3 --no-cache-dir install Keras==2.1.6
ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin: ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin:
WORKDIR /root WORKDIR /root
\ No newline at end of file
deployment/docker/Dockerfile.build.base deleted 100644 → 0
View file @ 684fc312
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
FROM nvidia/cuda:9.0-cudnn7-devel-ubuntu16.04
LABEL maintainer='Microsoft NNI Team<nni@microsoft.com>'
ENV HADOOP_VERSION=2.7.2
LABEL HADOOP_VERSION=2.7.2
RUN DEBIAN_FRONTEND=noninteractive && \
apt-get -y update && \
apt-get -y install sudo \
apt-utils \
git \
curl \
vim \
unzip \
wget \
build-essential \
cmake \
libopenblas-dev \
automake \
openjdk-8-jdk \
openssh-client \
openssh-server \
lsof \
python3.5 \
python3-dev \
python3-pip \
python3-tk \
libcupti-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# numpy 1.14.3 scipy 1.1.0
RUN pip3 --no-cache-dir install \
numpy==1.14.3 scipy==1.1.0
#
#Install hadoop
#
RUN wget -qO- http://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz | \
tar xz -C /usr/local && \
mv /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop
#
#Install NNI
#
RUN pip3 install --user nni-pkg
ENV JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 \
HADOOP_INSTALL=/usr/local/hadoop \
NVIDIA_VISIBLE_DEVICES=all
ENV HADOOP_PREFIX=${HADOOP_INSTALL} \
HADOOP_BIN_DIR=${HADOOP_INSTALL}/bin \
HADOOP_SBIN_DIR=${HADOOP_INSTALL}/sbin \
HADOOP_HDFS_HOME=${HADOOP_INSTALL} \
HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_INSTALL}/lib/native \
HADOOP_OPTS="-Djava.library.path=${HADOOP_INSTALL}/lib/native"
ENV PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/root/.local/bin:/usr/bin:/sbin:/bin:${HADOOP_BIN_DIR}:${HADOOP_SBIN_DIR} \
LD_LIBRARY_PATH=/usr/local/nvidia/lib:/usr/local/nvidia/lib64:/usr/local/cuda/lib64:/usr/local/cuda/targets/x86_64-linux/lib/stubs:${JAVA_HOME}/jre/lib/amd64/server
WORKDIR /root
docs/GetStarted.md
View file @ 17d85669
...@@ -15,13 +15,11 @@ ...@@ -15,13 +15,11 @@
* __Install NNI through pip__ * __Install NNI through pip__
python3 -m pip install --user nni-pkg python3 -m pip install --user nni-pkg
source ~/.bashrc
* __Install NNI through source code__ * __Install NNI through source code__
git clone -b v0.3 https://github.com/Microsoft/nni.git git clone -b v0.3 https://github.com/Microsoft/nni.git
cd nni cd nni
chmod +x install.sh
source install.sh source install.sh
## **Quick start: run a customized experiment** ## **Quick start: run a customized experiment**
......
src/sdk/pynni/setup.py
View file @ 17d85669
...@@ -27,7 +27,7 @@ def read(fname): ...@@ -27,7 +27,7 @@ def read(fname):
setuptools.setup( setuptools.setup(
name = 'nni', name = 'nni',
version = '0.2.0', version = '0.2.1',
packages = setuptools.find_packages(exclude=['tests']), packages = setuptools.find_packages(exclude=['tests']),
python_requires = '>=3.5', python_requires = '>=3.5',
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment