Unverified Commit b7e97992 authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Merge pull request #107 from Microsoft/master

merge master
parents 695d8664 892d3c1e
......@@ -46,7 +46,8 @@ NNI_YARN_FOLDER ?= /tmp/nni-yarn
NNI_YARN := PATH=$(BIN_FOLDER):$${PATH} $(NNI_YARN_FOLDER)/bin/yarn
## Version number
NNI_VERSION = $(shell git describe --tags)
NNI_VERSION_VALUE = $(shell git describe --tags)
NNI_VERSION_TEMPLATE = 999.0.0-developing
# Main targets
......@@ -160,18 +161,18 @@ install-dependencies: $(NNI_NODE_TARBALL) $(NNI_YARN_TARBALL)
.PHONY: install-python-modules
install-python-modules:
#$(_INFO) Installing Python SDK $(_END)
cd src/sdk/pynni && sed -ie 's/NNI_VERSION/$(NNI_VERSION)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) .
cd src/sdk/pynni && sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) .
#$(_INFO) Installing nnictl $(_END)
cd tools && sed -ie 's/NNI_VERSION/$(NNI_VERSION)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) .
cd tools && sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) .
.PHONY: dev-install-python-modules
dev-install-python-modules:
#$(_INFO) Installing Python SDK $(_END)
cd src/sdk/pynni && sed -ie 's/NNI_VERSION/$(NNI_VERSION)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) -e .
cd src/sdk/pynni && sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) -e .
#$(_INFO) Installing nnictl $(_END)
cd tools && sed -ie 's/NNI_VERSION/$(NNI_VERSION)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) -e .
cd tools && sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && $(PIP_INSTALL) $(PIP_MODE) -e .
.PHONY: install-node-modules
install-node-modules:
......@@ -179,7 +180,7 @@ install-node-modules:
rm -rf $(NNI_PKG_FOLDER)
cp -r src/nni_manager/dist $(NNI_PKG_FOLDER)
cp src/nni_manager/package.json $(NNI_PKG_FOLDER)
sed -ie 's/NNI_VERSION/$(NNI_VERSION)/' $(NNI_PKG_FOLDER)/package.json
sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(NNI_PKG_FOLDER)/package.json
$(NNI_YARN) --prod --cwd $(NNI_PKG_FOLDER)
cp -r src/webui/build $(NNI_PKG_FOLDER)/static
......@@ -189,7 +190,7 @@ dev-install-node-modules:
rm -rf $(NNI_PKG_FOLDER)
ln -sf ${PWD}/src/nni_manager/dist $(NNI_PKG_FOLDER)
cp src/nni_manager/package.json $(NNI_PKG_FOLDER)
sed -ie 's/NNI_VERSION/$(NNI_VERSION)/' $(NNI_PKG_FOLDER)/package.json
sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(NNI_PKG_FOLDER)/package.json
ln -sf ${PWD}/src/nni_manager/node_modules $(NNI_PKG_FOLDER)/node_modules
ln -sf ${PWD}/src/webui/build $(NNI_PKG_FOLDER)/static
......
......@@ -14,6 +14,10 @@ jobs:
- script: |
source install.sh
displayName: 'Install nni toolkit via source code'
- script: |
cd test
source unittest.sh
displayName: 'Unit test'
- script: |
cd test
PATH=$HOME/.local/bin:$PATH python3 naive_test.py
......@@ -21,7 +25,7 @@ jobs:
- script: |
cd test
PATH=$HOME/.local/bin:$PATH python3 sdk_test.py
displayName: 'Built-in tuner tests'
displayName: 'Built-in dispatcher tests'
- job: 'macOS_10_13'
pool:
......@@ -37,6 +41,10 @@ jobs:
- script: |
source install.sh
displayName: 'Install nni toolkit via source code'
- script: |
cd test
PATH=$HOME/Library/Python/3.7/bin:$PATH && source unittest.sh
displayName: 'Unit test'
- script: |
cd test
PATH=$HOME/Library/Python/3.7/bin:$PATH python3 naive_test.py
......@@ -44,4 +52,4 @@ jobs:
- script: |
cd test
PATH=$HOME/Library/Python/3.7/bin:$PATH python3 sdk_test.py
displayName: 'Built-in tuner tests'
\ No newline at end of file
displayName: 'Built-in dispatcher tests'
\ No newline at end of file
......@@ -11,7 +11,8 @@ else
$(error platform $(UNAME_S) not supported)
endif
NNI_VERSION = $(shell git describe --tags)
NNI_VERSION_VALUE = $(shell git describe --tags)
NNI_VERSION_TEMPLATE = 999.0.0-developing
.PHONY: build
build:
......@@ -26,10 +27,10 @@ build:
cp -r $(CWD)../../src/nni_manager/dist $(CWD)nni
cp -r $(CWD)../../src/webui/build $(CWD)nni/static
cp $(CWD)../../src/nni_manager/package.json $(CWD)nni
sed -ie 's/NNI_VERSION/$(NNI_VERSION)/' $(CWD)nni/package.json
sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' $(CWD)nni/package.json
cd $(CWD)nni && yarn --prod
cd $(CWD) && sed -ie 's/NNI_VERSION/$(NNI_VERSION)/' setup.py && python3 setup.py bdist_wheel -p $(WHEEL_SPEC)
cd $(CWD)../../src/sdk/pynni && sed -ie 's/NNI_VERSION/$(NNI_VERSION)/' setup.py && python3 setup.py bdist_wheel
cd $(CWD) && sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && python3 setup.py bdist_wheel -p $(WHEEL_SPEC)
cd $(CWD)../../src/sdk/pynni && sed -ie 's/$(NNI_VERSION_TEMPLATE)/$(NNI_VERSION_VALUE)/' setup.py && python3 setup.py bdist_wheel
cp -r $(CWD)../../src/sdk/pynni/dist/*.whl $(CWD)dist
cd $(CWD)
......
......@@ -20,7 +20,7 @@ with open('../../README.md', 'r') as fh:
setuptools.setup(
name = 'nni',
version = 'NNI_VERSION',
version = '999.0.0-developing',
author = 'Microsoft NNI team',
author_email = 'nni@microsoft.com',
description = 'Neural Network Intelligence package',
......
......@@ -10,7 +10,7 @@ For now, NNI has supported the following tuner algorithms. Note that NNI install
- [Batch Tuner](#Batch)
- [Grid Search](#Grid)
- [Hyperband](#Hyperband)
- [Network Morphism](#NetworkMorphism)
- [Network Morphism](#NetworkMorphism) (require pyTorch)
## Supported tuner algorithms
......@@ -180,6 +180,10 @@ _Usage_:
[Network Morphism](7) provides functions to automatically search for architecture of deep learning models. Every child network inherits the knowledge from its parent network and morphs into diverse types of networks, including changes of depth, width and skip-connection. Next, it estimates the value of child network using the history architecture and metric pairs. Then it selects the most promising one to train. More detail can be referred to [here](../src/sdk/pynni/nni/networkmorphism_tuner/README.md).
_Installation_:
NetworkMorphism requires [pyTorch](https://pytorch.org/get-started/locally), so users should install it first.
_Suggested scenario_: It is suggested that you want to apply deep learning methods to your task (your own dataset) but you have no idea of how to choose or design a network. You modify the [example](../examples/trials/network_morphism/cifar10/cifar10_keras.py) to fit your own dataset and your own data augmentation method. Also you can change the batch size, learning rate or optimizer. It is feasible for different tasks to find a good network architecture. Now this tuner only supports the cv domain.
_Usage_:
......
......@@ -35,7 +35,7 @@ class CustomInstallCommand(install):
setup(
name = 'nni',
version = 'NNI_VERSION',
version = '999.0.0-developing',
author = 'Microsoft NNI Team',
author_email = 'nni@microsoft.com',
description = 'Neural Network Intelligence project',
......
......@@ -120,7 +120,7 @@ class Logger {
private log(level: string, param: any[]): void {
const buffer: WritableStreamBuffer = new WritableStreamBuffer();
buffer.write(`[${(new Date()).toISOString()}] ${level} `);
buffer.write(format.apply(null, param));
buffer.write(format(null, param));
buffer.write('\n');
buffer.end();
this.bufferSerialEmitter.feed(buffer.getContents());
......
......@@ -21,7 +21,7 @@
import * as assert from 'assert';
import * as cpp from 'child-process-promise';
import { ChildProcess, spawn } from 'child_process';
import { ChildProcess, spawn, StdioOptions } from 'child_process';
import { Deferred } from 'ts-deferred';
import * as component from '../common/component';
import { DataStore, MetricDataRecord, MetricType, TrialJobInfo } from '../common/datastore';
......@@ -35,7 +35,7 @@ import {
import {
TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, TrialJobStatus
} from '../common/trainingService';
import { delay , getLogDir, getCheckpointDir, getMsgDispatcherCommand, mkDirP} from '../common/utils';
import { delay, getLogDir, getCheckpointDir, getMsgDispatcherCommand, mkDirP } from '../common/utils';
import {
ADD_CUSTOMIZED_TRIAL_JOB, INITIALIZE, INITIALIZED, KILL_TRIAL_JOB, NEW_TRIAL_JOB, NO_MORE_TRIAL_JOBS,
REPORT_METRIC_DATA, REQUEST_TRIAL_JOBS, SEND_TRIAL_JOB_PARAMETER, TERMINATE, TRIAL_END, UPDATE_SEARCH_SPACE
......@@ -253,7 +253,7 @@ class NNIManager implements Manager {
if (this.dispatcher !== undefined) {
return;
}
const stdio: (string | NodeJS.WriteStream)[] = ['ignore', process.stdout, process.stderr, 'pipe', 'pipe'];
const stdio: StdioOptions = ['ignore', process.stdout, process.stderr, 'pipe', 'pipe'];
let newCwd: string;
if (cwd === undefined || cwd === '') {
newCwd = getLogDir();
......@@ -387,7 +387,8 @@ class NNIManager implements Manager {
this.dispatcher.sendCommand(TRIAL_END, JSON.stringify({
trial_job_id: trialJobDetail.id,
event: trialJobDetail.status,
hyper_params: hyperParams }));
hyper_params: hyperParams
}));
break;
case 'FAILED':
case 'SYS_CANCELED':
......@@ -403,7 +404,8 @@ class NNIManager implements Manager {
this.dispatcher.sendCommand(TRIAL_END, JSON.stringify({
trial_job_id: trialJobDetail.id,
event: trialJobDetail.status,
hyper_params: hyperParams}));
hyper_params: hyperParams
}));
break;
case 'WAITING':
case 'RUNNING':
......
......@@ -20,15 +20,15 @@
'use strict';
import * as assert from 'assert';
import { ChildProcess, spawn } from 'child_process';
import { ChildProcess, spawn, StdioOptions } from 'child_process';
import { Deferred } from 'ts-deferred';
import { cleanupUnitTest, prepareUnitTest } from '../../common/utils';
import * as CommandType from '../commands';
import { createDispatcherInterface, IpcInterface } from '../ipcInterface';
import { NNIError } from '../../common/errors';
let sentCommands: {[key: string]: string}[] = [];
const receivedCommands: {[key: string]: string}[] = [];
let sentCommands: { [key: string]: string }[] = [];
const receivedCommands: { [key: string]: string }[] = [];
let commandTooLong: Error | undefined;
let rejectCommandType: Error | undefined;
......@@ -38,7 +38,7 @@ function runProcess(): Promise<Error | null> {
const deferred: Deferred<Error | null> = new Deferred<Error | null>();
// create fake assessor process
const stdio: {}[] = ['ignore', 'pipe', process.stderr, 'pipe', 'pipe'];
const stdio: StdioOptions = ['ignore', 'pipe', process.stderr, 'pipe', 'pipe'];
const proc: ChildProcess = spawn('python3 assessor.py', [], { stdio, cwd: 'core/test', shell: true });
// record its sent/received commands on exit
......
......@@ -20,7 +20,7 @@
'use strict';
import * as assert from 'assert';
import { ChildProcess, spawn } from 'child_process';
import { ChildProcess, spawn, StdioOptions } from 'child_process';
import { Deferred } from 'ts-deferred';
import { cleanupUnitTest, prepareUnitTest, getMsgDispatcherCommand } from '../../common/utils';
import * as CommandType from '../commands';
......@@ -32,9 +32,9 @@ let procError: boolean = false;
function startProcess(): void {
// create fake assessor process
const stdio: {}[] = ['ignore', 'pipe', process.stderr, 'pipe', 'pipe'];
const stdio: StdioOptions = ['ignore', 'pipe', process.stderr, 'pipe', 'pipe'];
const dispatcherCmd : string = getMsgDispatcherCommand(
const dispatcherCmd: string = getMsgDispatcherCommand(
// Mock tuner config
{
className: 'DummyTuner',
......
{
"name": "nni",
"version": "NNI_VERSION",
"version": "999.0.0-developing",
"main": "index.js",
"scripts": {
"postbuild": "cp -rf scripts ./dist/ && cp -rf config ./dist/",
"build": "tsc",
"test": "mocha -r ts-node/register -t 15000 --recursive **/*.test.ts --exclude node_modules/**/**/*.test.ts --colors",
"start": "node dist/main.js"
"test": "mocha -r ts-node/register -t 15000 --recursive **/*.test.ts --exclude node_modules/**/**/*.test.ts --exclude core/test/nnimanager.test.ts --colors",
"start": "node dist/main.js",
"tslint": "tslint -p ."
},
"license": "MIT",
"dependencies": {
......@@ -24,7 +25,7 @@
"ts-deferred": "^1.0.4",
"typescript-ioc": "^1.2.4",
"typescript-string-operations": "^1.3.1",
"webhdfs":"^1.2.0",
"webhdfs": "^1.2.0",
"azure-storage": "^2.10.2",
"kubernetes-client": "^6.5.0",
"js-base64": "^2.4.9"
......@@ -34,7 +35,7 @@
"@types/chai-as-promised": "^7.1.0",
"@types/express": "^4.16.0",
"@types/mocha": "^5.2.5",
"@types/node": "^10.5.5",
"@types/node": "^10.12.18",
"@types/request": "^2.47.1",
"@types/rx": "^4.1.1",
"@types/sqlite3": "^3.1.3",
......@@ -47,9 +48,9 @@
"rmdir": "^1.2.0",
"tmp": "^0.0.33",
"ts-node": "^7.0.0",
"tslint": "^5.11.0",
"tslint-microsoft-contrib": "^5.1.0",
"typescript": "^3.0.1"
"tslint": "^5.12.0",
"tslint-microsoft-contrib": "^6.0.0",
"typescript": "^3.2.2"
},
"engines": {
"node": ">=10.0.0"
......
......@@ -27,7 +27,7 @@ def read(fname):
setuptools.setup(
name = 'nni-sdk',
version = 'NNI_VERSION',
version = '999.0.0-developing',
packages = setuptools.find_packages(exclude=['tests']),
python_requires = '>=3.5',
......
......@@ -49,7 +49,7 @@ class ProtocolTestCase(TestCase):
_prepare_send()
exception = None
try:
send(CommandType.NewTrialJob, ' ' * 1_000_000)
send(CommandType.NewTrialJob, ' ' * 1000000)
except AssertionError as e:
exception = e
self.assertIsNotNone(exception)
......
......@@ -18,7 +18,6 @@
# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# ==================================================================================================
import nni
import nni.platform.test as test_platform
import nni.trial
......@@ -30,11 +29,18 @@ from unittest import TestCase, main
class TrialTestCase(TestCase):
def setUp(self):
self._trial_params = { 'msg': 'hi', 'x': 123, 'dict': { 'key': 'value', 'y': None } }
nni.trial._params = { 'parameter_id': 'test_param', 'parameters': self._trial_params }
test_platform._params = { 'parameter_id': 'test_param', 'parameters': self._trial_params }
def test_get_next_parameter(self):
self.assertEqual(nni.get_next_parameter(), self._trial_params)
def test_get_current_parameter(self):
nni.get_next_parameter()
self.assertEqual(nni.get_current_parameter('x'), 123)
def test_get_sequence_id(self):
self.assertEqual(nni.get_sequence_id(), 0)
def test_report_intermediate_result(self):
nni.report_intermediate_result(123)
self.assertEqual(test_platform.get_last_metric(), {
......
#!/bin/bash
CWD=${PWD}
# -------------For python unittest-------------
## ------Run annotation test------
echo ""
echo "===========================Testing: nni_annotation==========================="
cd ${CWD}/../tools/
python3 -m unittest -v nni_annotation/test_annotation.py
## Export certain environment variables for unittest code to work
export NNI_TRIAL_JOB_ID=test_trial_job_id
export NNI_PLATFORM=unittest
## ------Run sdk test------
echo ""
echo "===========================Testing: nni_sdk==========================="
cd ${CWD}/../src/sdk/pynni/
python3 -m unittest discover -v tests
# -------------For typescrip unittest-------------
cd ${CWD}/../src/nni_manager
echo ""
echo "===========================Testing: nni_manager==========================="
npm run test
\ No newline at end of file
......@@ -2,7 +2,7 @@ import setuptools
setuptools.setup(
name = 'nnictl',
version = 'NNI_VERSION',
version = '999.0.0-developing',
packages = setuptools.find_packages(exclude=['*test*']),
python_requires = '>=3.5',
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment