Unverified Commit 0fb78620 authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Merge pull request #238 from microsoft/master

merge master
parents 3ee09617 b8d19e45
......@@ -47,9 +47,9 @@ jobs:
fi
echo "TEST_IMG:$TEST_IMG"
cd test
python3 generate_ts_config.py --ts kubeflow --keyvault_vaultname $(keyVault_vaultName) --keyvault_name $(keyVault_name) \
python3 nni_test/nnitest/generate_ts_config.py --ts kubeflow --keyvault_vaultname $(keyVault_vaultName) --keyvault_name $(keyVault_name) \
--azs_account $(azureStorage_accountName) --azs_share $(azureStorage_azureShare) --nni_docker_image $TEST_IMG --nni_manager_ip $(nni_manager_ip)
cat training_service.yml
PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts kubeflow --exclude multi_phase
cat config/training_service.yml
PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts kubeflow --exclude multi-phase
displayName: 'integration test'
......@@ -9,34 +9,14 @@ jobs:
- script: |
python -m pip install scikit-learn==0.20.0 --user
python -m pip install keras==2.1.6 --user
python -m pip install torch===1.2.0 torchvision===0.4.1 -f https://download.pytorch.org/whl/torch_stable.html --user
python -m pip install torchvision===0.4.1 torch===1.3.1 -f https://download.pytorch.org/whl/torch_stable.html --user
python -m pip install tensorflow-gpu==1.11.0 --user
displayName: 'Install dependencies for integration tests'
- script: |
cd test
python generate_ts_config.py --ts local
displayName: 'generate config files'
- script: |
cd test
python config_test.py --ts local --local_gpu --exclude smac,bohb
displayName: 'Examples and advanced features tests on local machine'
- script: |
cd test
powershell.exe -file unittest.ps1
powershell.exe -file scripts/unittest.ps1
displayName: 'unit test'
- script: |
cd test
python naive_test.py
displayName: 'Naive test'
- script: |
cd test
python tuner_test.py
displayName: 'Built-in tuners / assessors tests'
- script: |
cd test
python metrics_test.py
displayName: 'Trial job metrics test'
- script: |
cd test
PATH=$HOME/.local/bin:$PATH python3 cli_test.py
displayName: 'nnicli test'
python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts local
displayName: 'Integration tests'
......@@ -10,8 +10,8 @@ jobs:
displayName: 'Install nni toolkit via source code'
- script: |
python3 -m pip install scikit-learn==0.20.0 --user
python3 -m pip install torch==1.2.0 --user
python3 -m pip install torchvision==0.4.0 --user
python3 -m pip install torchvision==0.4.1 --user
python3 -m pip install torch==1.3.1 --user
python3 -m pip install keras==2.1.6 --user
python3 -m pip install tensorflow-gpu==1.15 --user
sudo apt-get install swig -y
......@@ -20,25 +20,13 @@ jobs:
displayName: 'Install dependencies for integration tests'
- script: |
cd test
source unittest.sh
source scripts/unittest.sh
displayName: 'Unit test'
- script: |
cd test
PATH=$HOME/.local/bin:$PATH python3 naive_test.py
displayName: 'Naive test'
PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts local
displayName: 'Integration test'
- script: |
cd test
PATH=$HOME/.local/bin:$PATH python3 tuner_test.py
displayName: 'Built-in tuners / assessors tests'
- script: |
cd test
PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts local --local_gpu
displayName: 'Examples and advanced features tests on local machine'
- script: |
cd test
PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
displayName: 'Trial job metrics test'
- script: |
cd test
PATH=$HOME/.local/bin:$PATH python3 cli_test.py
displayName: 'nnicli test'
source scripts/model_compression.sh
displayName: 'Model compression test'
......@@ -63,7 +63,6 @@ jobs:
cd test
set PATH=$(ENV_PATH)
python --version
python generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --pai_pwd $(pai_pwd) --vc $(pai_virtual_cluster) --nni_docker_image $(docker_image) --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip)
python config_test.py --ts pai --exclude multi_phase,smac,bohb
python nni_test/nnitest/generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --pai_pwd $(pai_pwd) --vc $(pai_virtual_cluster) --nni_docker_image $(docker_image) --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip)
python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts pai --exclude multi-phase
displayName: 'Examples and advanced features tests on pai'
\ No newline at end of file
......@@ -51,9 +51,7 @@ jobs:
echo "TEST_IMG:$TEST_IMG"
cd test
python3 generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --nni_docker_image $TEST_IMG --pai_storage_plugin $(pai_storage_plugin)\
python3 nni_test/nnitest/generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --nni_docker_image $TEST_IMG --pai_storage_plugin $(pai_storage_plugin)\
--pai_token $(pai_token) --nni_manager_nfs_mount_path $(nni_manager_nfs_mount_path) --container_nfs_mount_path $(container_nfs_mount_path) --nni_manager_ip $(nni_manager_ip)
PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts pai --exclude multi_phase
PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts pai --exclude multi-phase
displayName: 'integration test'
......@@ -29,7 +29,7 @@ jobs:
inputs:
sshEndpoint: $(end_point)
runOptions: commands
commands: python3 /tmp/nnitest/$(Build.BuildId)/nni-remote/test/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni --os windows
commands: python3 /tmp/nnitest/$(Build.BuildId)/nni-remote/test/nni_test/nnitest/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni --os windows
displayName: 'Start docker'
- powershell: |
Write-Host "Downloading Putty..."
......@@ -42,15 +42,15 @@ jobs:
displayName: 'Get docker port'
- powershell: |
cd test
python generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
Get-Content training_service.yml
python config_test.py --ts remote --exclude cifar10,smac,bohb
python nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
Get-Content config/training_service.yml
python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote --exclude cifar10
displayName: 'integration test'
- task: SSH@0
inputs:
sshEndpoint: $(end_point)
runOptions: commands
commands: python3 /tmp/nnitest/$(Build.BuildId)/nni-remote/test/remote_docker.py --mode stop --name $(Build.BuildId) --os windows
commands: python3 /tmp/nnitest/$(Build.BuildId)/nni-remote/test/nni_test/nnitest/remote_docker.py --mode stop --name $(Build.BuildId) --os windows
displayName: 'Stop docker'
- task: SSH@0
inputs:
......
......@@ -39,7 +39,7 @@ jobs:
inputs:
sshEndpoint: $(end_point)
runOptions: commands
commands: python3 /tmp/nnitest/$(Build.BuildId)/test/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni
commands: python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni
displayName: 'Start docker'
- task: DownloadSecureFile@1
inputs:
......@@ -53,15 +53,14 @@ jobs:
- script: |
set -e
cd test
python3 generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
python3 nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
--remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
cat training_service.yml
PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts remote --exclude cifar10
PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
cat config/training_service.yml
PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
displayName: 'integration test'
- task: SSH@0
inputs:
sshEndpoint: $(end_point)
runOptions: commands
commands: python3 /tmp/nnitest/$(Build.BuildId)/test/remote_docker.py --mode stop --name $(Build.BuildId)
commands: python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode stop --name $(Build.BuildId)
displayName: 'Stop docker'
#!/bin/bash
set -e
CWD=${PWD}
echo ""
echo "===========================Testing: pruning and speedup==========================="
cd ${CWD}/../examples/model_compress
echo "testing slim pruning and speedup..."
python3 model_prune_torch.py --pruner_name slim --pretrain_epochs 1 --prune_epochs 1
python3 model_speedup.py --example_name slim --model_checkpoint ./checkpoints/pruned_vgg19_cifar10_slim.pth \
--masks_file ./checkpoints/mask_vgg19_cifar10_slim.pth
echo "testing l1 pruning and speedup..."
python3 model_prune_torch.py --pruner_name l1 --pretrain_epochs 1 --prune_epochs 1
python3 model_speedup.py --example_name l1filter --model_checkpoint ./checkpoints/pruned_vgg16_cifar10_l1.pth \
--masks_file ./checkpoints/mask_vgg16_cifar10_l1.pth
echo "testing apoz pruning and speedup..."
python3 model_prune_torch.py --pruner_name apoz --pretrain_epochs 1 --prune_epochs 1
python3 model_speedup.py --example_name apoz --model_checkpoint ./checkpoints/pruned_vgg16_cifar10_apoz.pth \
--masks_file ./checkpoints/mask_vgg16_cifar10_apoz.pth
for name in level fpgm mean_activation
do
echo "testing $name pruning..."
python3 model_prune_torch.py --pruner_name $name --pretrain_epochs 1 --prune_epochs 1
done
#echo "testing lottery ticket pruning..."
#python3 lottery_torch_mnist_fc.py
echo ""
echo "===========================Testing: quantizers==========================="
# to be enabled
#echo "testing QAT quantizer..."
#python3 QAT_torch_quantizer.py
#echo "testing DoReFa quantizer..."
#python3 DoReFaQuantizer_torch_mnist.py
#echo "testing BNN quantizer..."
#python3 BNN_quantizer_cifar10.py
rm -rf ./checkpoints/*
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
import sys
import os.path as osp
import subprocess
import sys
import time
import traceback
from utils import get_yml_content, dump_yml_content, setup_experiment, get_nni_log_path, is_experiment_done
from utils import GREEN, RED, CLEAR, EXPERIMENT_URL
TUNER_LIST = ['GridSearch', 'BatchTuner', 'TPE', 'Random', 'Anneal', 'Evolution']
ASSESSOR_LIST = ['Medianstop']
def get_config_file_path():
if sys.platform == 'win32':
config_file = osp.join('tuner_test', 'local_win32.yml')
else:
config_file = osp.join('tuner_test', 'local.yml')
return config_file
def switch(dispatch_type, dispatch_name):
'''Change dispatch in config.yml'''
config_path = get_config_file_path()
experiment_config = get_yml_content(config_path)
if dispatch_name in ['GridSearch', 'BatchTuner', 'Random']:
experiment_config[dispatch_type.lower()] = {
'builtin' + dispatch_type + 'Name': dispatch_name
}
else:
experiment_config[dispatch_type.lower()] = {
'builtin' + dispatch_type + 'Name': dispatch_name,
'classArgs': {
'optimize_mode': 'maximize'
}
}
if dispatch_name == 'BatchTuner':
experiment_config['searchSpacePath'] = 'batchtuner_search_space.json'
else:
experiment_config['searchSpacePath'] = 'search_space.json'
dump_yml_content(config_path, experiment_config)
def test_builtin_dispatcher(dispatch_type, dispatch_name):
'''test a dispatcher whose type is dispatch_type and name is dispatch_name'''
switch(dispatch_type, dispatch_name)
print('Testing %s...' % dispatch_name)
proc = subprocess.run(['nnictl', 'create', '--config', get_config_file_path()])
assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
nnimanager_log_path = get_nni_log_path(EXPERIMENT_URL)
for _ in range(20):
time.sleep(3)
# check if experiment is done
experiment_status = is_experiment_done(nnimanager_log_path)
if experiment_status:
break
assert experiment_status, 'Failed to finish in 1 min'
def run(dispatch_type):
'''test all dispatchers whose type is dispatch_type'''
assert dispatch_type in ['Tuner', 'Assessor'], 'Unsupported dispatcher type: %s' % (dispatch_type)
dipsatcher_list = TUNER_LIST if dispatch_type == 'Tuner' else ASSESSOR_LIST
for dispatcher_name in dipsatcher_list:
try:
# Sleep here to make sure previous stopped exp has enough time to exit to avoid port conflict
time.sleep(6)
test_builtin_dispatcher(dispatch_type, dispatcher_name)
print(GREEN + 'Test %s %s: TEST PASS' % (dispatcher_name, dispatch_type) + CLEAR)
except Exception as error:
print(RED + 'Test %s %s: TEST FAIL' % (dispatcher_name, dispatch_type) + CLEAR)
print('%r' % error)
traceback.print_exc()
raise error
finally:
subprocess.run(['nnictl', 'stop'])
if __name__ == '__main__':
installed = (sys.argv[-1] != '--preinstall')
setup_experiment(installed)
run('Tuner')
run('Assessor')
{
"combine_params":
{
"_type" : "choice",
"_value" : [{"x": 1},
{"x": 100}]
}
}
assessor:
builtinAssessorName: Medianstop
classArgs:
optimize_mode: maximize
authorName: nni
experimentName: test_sdk
maxExecDuration: 1h
maxTrialNum: 2
searchSpacePath: search_space.json
trainingServicePlatform: local
trial:
codeDir: .
command: python3 naive_trial.py
gpuNum: 0
trialConcurrency: 2
tuner:
builtinTunerName: Evolution
classArgs:
optimize_mode: maximize
useAnnotation: false
assessor:
builtinAssessorName: Medianstop
classArgs:
optimize_mode: maximize
authorName: nni
experimentName: test_sdk
maxExecDuration: 1h
maxTrialNum: 2
searchSpacePath: search_space.json
trainingServicePlatform: local
trial:
codeDir: .
command: python naive_trial.py
gpuNum: 0
trialConcurrency: 2
tuner:
builtinTunerName: Evolution
classArgs:
optimize_mode: maximize
useAnnotation: false
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
import nni
params = nni.get_next_parameter()
print('params:', params)
x = params['x']
nni.report_final_result(x)
......@@ -2,6 +2,7 @@
# Licensed under the MIT license.
import os
import site
import sys
import json
import socket
......@@ -74,3 +75,44 @@ def get_python_dir(sitepackages_path):
return str(Path(sitepackages_path))
else:
return str(Path(sitepackages_path).parents[2])
def get_nni_installation_path():
''' Find nni lib from the following locations in order
Return nni root directory if it exists
'''
def try_installation_path_sequentially(*sitepackages):
'''Try different installation path sequentially util nni is found.
Return None if nothing is found
'''
def _generate_installation_path(sitepackages_path):
python_dir = get_python_dir(sitepackages_path)
entry_file = os.path.join(python_dir, 'nni', 'main.js')
if os.path.isfile(entry_file):
return python_dir
return None
for sitepackage in sitepackages:
python_dir = _generate_installation_path(sitepackage)
if python_dir:
return python_dir
return None
if os.getenv('VIRTUAL_ENV'):
# if 'virtualenv' package is used, `site` has not attr getsitepackages, so we will instead use VIRTUAL_ENV
# Note that conda venv will not have VIRTUAL_ENV
python_dir = os.getenv('VIRTUAL_ENV')
else:
python_sitepackage = site.getsitepackages()[0]
# If system-wide python is used, we will give priority to using `local sitepackage`--"usersitepackages()" given
# that nni exists there
if python_sitepackage.startswith('/usr') or python_sitepackage.startswith('/Library'):
python_dir = try_installation_path_sequentially(site.getusersitepackages(), site.getsitepackages()[0])
else:
python_dir = try_installation_path_sequentially(site.getsitepackages()[0], site.getusersitepackages())
if python_dir:
entry_file = os.path.join(python_dir, 'nni', 'main.js')
if os.path.isfile(entry_file):
return os.path.join(python_dir, 'nni')
print_error('Fail to find nni under python library')
exit(1)
\ No newline at end of file
......@@ -6,7 +6,6 @@ import os
import sys
import string
import random
import site
import time
import tempfile
from subprocess import Popen, check_call, CalledProcessError, PIPE, STDOUT
......@@ -17,7 +16,7 @@ from .rest_utils import rest_put, rest_post, check_rest_server, check_response
from .url_utils import cluster_metadata_url, experiment_url, get_local_urls
from .config_utils import Config, Experiments
from .common_utils import get_yml_content, get_json_content, print_error, print_normal, \
detect_port, get_user, get_python_dir
detect_port, get_user, get_nni_installation_path
from .constants import NNICTL_HOME_DIR, ERROR_INFO, REST_TIME_OUT, EXPERIMENT_SUCCESS_INFO, LOG_HEADER, PACKAGE_REQUIREMENTS
from .command_utils import check_output_command, kill_command
from .nnictl_utils import update_experiment
......@@ -37,47 +36,6 @@ def print_log_content(config_file_name):
print_normal(' Stderr:')
print(check_output_command(stderr_full_path))
def get_nni_installation_path():
''' Find nni lib from the following locations in order
Return nni root directory if it exists
'''
def try_installation_path_sequentially(*sitepackages):
'''Try different installation path sequentially util nni is found.
Return None if nothing is found
'''
def _generate_installation_path(sitepackages_path):
python_dir = get_python_dir(sitepackages_path)
entry_file = os.path.join(python_dir, 'nni', 'main.js')
if os.path.isfile(entry_file):
return python_dir
return None
for sitepackage in sitepackages:
python_dir = _generate_installation_path(sitepackage)
if python_dir:
return python_dir
return None
if os.getenv('VIRTUAL_ENV'):
# if 'virtualenv' package is used, `site` has not attr getsitepackages, so we will instead use VIRTUAL_ENV
# Note that conda venv will not have VIRTUAL_ENV
python_dir = os.getenv('VIRTUAL_ENV')
else:
python_sitepackage = site.getsitepackages()[0]
# If system-wide python is used, we will give priority to using `local sitepackage`--"usersitepackages()" given
# that nni exists there
if python_sitepackage.startswith('/usr') or python_sitepackage.startswith('/Library'):
python_dir = try_installation_path_sequentially(site.getusersitepackages(), site.getsitepackages()[0])
else:
python_dir = try_installation_path_sequentially(site.getsitepackages()[0], site.getusersitepackages())
if python_dir:
entry_file = os.path.join(python_dir, 'nni', 'main.js')
if os.path.isfile(entry_file):
return os.path.join(python_dir, 'nni')
print_error('Fail to find nni under python library')
exit(1)
def start_rest_server(port, platform, mode, config_file_name, foreground=False, experiment_id=None, log_dir=None, log_level=None):
'''Run nni manager process'''
if detect_port(port):
......
......@@ -19,7 +19,7 @@ from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url, export_
from .config_utils import Config, Experiments
from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \
EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT
from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content
from .common_utils import print_normal, print_error, print_warning, detect_process, get_yml_content, get_nni_installation_path
from .command_utils import check_output_command, kill_command
from .ssh_utils import create_ssh_sftp_client, remove_remote_directory
......@@ -392,10 +392,13 @@ def webui_url(args):
def webui_nas(args):
'''launch nas ui'''
print_normal('Starting NAS UI...')
# TODO: find file path on installing with pypi
# TODO: use correct node on win32
try:
cmds = ['node', 'src/nasui/server.js', '--port', str(args.port), '--logdir', args.logdir]
entry_dir = get_nni_installation_path()
entry_file = os.path.join(entry_dir, 'nasui', 'server.js')
node_command = 'node'
if sys.platform == 'win32':
node_command = os.path.join(entry_dir[:-3], 'Scripts', 'node.exe')
cmds = [node_command, '--max-old-space-size=4096', entry_file, '--port', str(args.port), '--logdir', args.logdir]
subprocess.run(cmds)
except KeyboardInterrupt:
pass
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment