Refactor integration tests (#2190)

test/pipelines-it-local.yml → test/pipelines/pipelines-it-local.yml

@@ -10,8 +10,8 @@ jobs:
     displayName: 'Install nni toolkit via source code'
   - script: |
       python3 -m pip install scikit-learn==0.20.0 --user
-      python3 -m pip install torch==1.2.0 --user
-      python3 -m pip install torchvision==0.4.0 --user
+      python3 -m pip install torchvision==0.4.1 --user
+      python3 -m pip install torch==1.3.1 --user
       python3 -m pip install keras==2.1.6 --user
       python3 -m pip install tensorflow-gpu==1.15 --user
       sudo apt-get install swig -y
@@ -20,25 +20,13 @@ jobs:
     displayName: 'Install dependencies for integration tests'
   - script: |
       cd test
-      source unittest.sh
+      source scripts/unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 naive_test.py
-    displayName: 'Naive test'
+      PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts local
+    displayName: 'Integration test'
   - script: |
       cd test
-      PATH=$HOME/.local/bin:$PATH python3 tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
-  - script: |
-      cd test
-      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts local --local_gpu
-    displayName: 'Examples and advanced features tests on local machine'
-  - script: |
-      cd test
-      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
-    displayName: 'Trial job metrics test'
-  - script: |
-      cd test
-      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
-    displayName: 'nnicli test'
+      source scripts/model_compression.sh
+    displayName: 'Model compression test'

test/pipelines-it-pai-windows.yml → test/pipelines/pipelines-it-pai-windows.yml

@@ -63,7 +63,6 @@ jobs:
       cd test
       set PATH=$(ENV_PATH)
       python --version
-      python generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --pai_pwd $(pai_pwd) --vc $(pai_virtual_cluster) --nni_docker_image $(docker_image) --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip)
-
-      python config_test.py --ts pai --exclude multi_phase,smac,bohb
+      python nni_test/nnitest/generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --pai_pwd $(pai_pwd) --vc $(pai_virtual_cluster) --nni_docker_image $(docker_image) --data_dir $(data_dir) --output_dir $(output_dir) --nni_manager_ip $(nni_manager_ip)
+      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts pai --exclude multi-phase
     displayName: 'Examples and advanced features tests on pai'
\ No newline at end of file

test/pipelines-it-pai.yml → test/pipelines/pipelines-it-pai.yml

@@ -51,9 +51,7 @@ jobs:
 
       echo "TEST_IMG:$TEST_IMG"
       cd test
-      python3 generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --nni_docker_image $TEST_IMG --pai_storage_plugin $(pai_storage_plugin)\
+      python3 nni_test/nnitest/generate_ts_config.py --ts pai --pai_host $(pai_host) --pai_user $(pai_user) --nni_docker_image $TEST_IMG --pai_storage_plugin $(pai_storage_plugin)\
        --pai_token $(pai_token) --nni_manager_nfs_mount_path $(nni_manager_nfs_mount_path) --container_nfs_mount_path $(container_nfs_mount_path) --nni_manager_ip $(nni_manager_ip)
-
-      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts pai --exclude multi_phase
-      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
+      PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts pai --exclude multi-phase
     displayName: 'integration test'

test/pipelines-it-remote-windows.yml → test/pipelines/pipelines-it-remote-windows.yml

@@ -29,7 +29,7 @@ jobs:
     inputs:
       sshEndpoint: $(end_point)
       runOptions: commands
-      commands: python3 /tmp/nnitest/$(Build.BuildId)/nni-remote/test/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni --os windows
+      commands: python3 /tmp/nnitest/$(Build.BuildId)/nni-remote/test/nni_test/nnitest/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni --os windows
     displayName: 'Start docker'
   - powershell: |
       Write-Host "Downloading Putty..."
@@ -42,15 +42,15 @@ jobs:
     displayName: 'Get docker port'
   - powershell: |
       cd test
-      python generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
-      Get-Content training_service.yml
-      python config_test.py --ts remote --exclude cifar10,smac,bohb
+      python nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) --remote_port $(Get-Content port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
+      Get-Content config/training_service.yml
+      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote --exclude cifar10
     displayName: 'integration test'
   - task: SSH@0
     inputs:
       sshEndpoint: $(end_point)
       runOptions: commands
-      commands: python3 /tmp/nnitest/$(Build.BuildId)/nni-remote/test/remote_docker.py --mode stop --name $(Build.BuildId) --os windows
+      commands: python3 /tmp/nnitest/$(Build.BuildId)/nni-remote/test/nni_test/nnitest/remote_docker.py --mode stop --name $(Build.BuildId) --os windows
     displayName: 'Stop docker'
   - task: SSH@0
     inputs:

test/pipelines-it-remote.yml → test/pipelines/pipelines-it-remote.yml

@@ -39,7 +39,7 @@ jobs:
     inputs:
       sshEndpoint: $(end_point)
       runOptions: commands
-      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni
+      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni
     displayName: 'Start docker'
   - task: DownloadSecureFile@1
     inputs:
@@ -53,15 +53,14 @@ jobs:
   - script: |
       set -e
       cd test
-      python3 generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
+      python3 nni_test/nnitest/generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
       --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
-      cat training_service.yml
-      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts remote --exclude cifar10
-      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
+      cat config/training_service.yml
+      PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
     displayName: 'integration test'
   - task: SSH@0
     inputs:
       sshEndpoint: $(end_point)
       runOptions: commands
-      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/remote_docker.py --mode stop --name $(Build.BuildId)
+      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode stop --name $(Build.BuildId)
     displayName: 'Stop docker'

test/scripts/model_compression.sh

+#!/bin/bash
+set -e
+CWD=${PWD}
+
+echo ""
+echo "===========================Testing: pruning and speedup==========================="
+cd ${CWD}/../examples/model_compress
+
+echo "testing slim pruning and speedup..."
+python3 model_prune_torch.py --pruner_name slim --pretrain_epochs 1 --prune_epochs 1
+python3 model_speedup.py --example_name slim --model_checkpoint ./checkpoints/pruned_vgg19_cifar10_slim.pth \
+    --masks_file ./checkpoints/mask_vgg19_cifar10_slim.pth
+
+echo "testing l1 pruning and speedup..."
+python3 model_prune_torch.py --pruner_name l1 --pretrain_epochs 1 --prune_epochs 1
+python3 model_speedup.py --example_name l1filter --model_checkpoint ./checkpoints/pruned_vgg16_cifar10_l1.pth \
+    --masks_file ./checkpoints/mask_vgg16_cifar10_l1.pth
+
+echo "testing apoz pruning and speedup..."
+python3 model_prune_torch.py --pruner_name apoz --pretrain_epochs 1 --prune_epochs 1
+python3 model_speedup.py --example_name apoz --model_checkpoint ./checkpoints/pruned_vgg16_cifar10_apoz.pth \
+    --masks_file ./checkpoints/mask_vgg16_cifar10_apoz.pth
+
+for name in level fpgm mean_activation
+do
+    echo "testing $name pruning..."
+    python3 model_prune_torch.py --pruner_name $name --pretrain_epochs 1 --prune_epochs 1
+done
+
+#echo "testing lottery ticket pruning..."
+#python3 lottery_torch_mnist_fc.py
+
+echo ""
+echo "===========================Testing: quantizers==========================="
+# to be enabled
+#echo "testing QAT quantizer..."
+#python3 QAT_torch_quantizer.py
+
+#echo "testing DoReFa quantizer..."
+#python3 DoReFaQuantizer_torch_mnist.py
+
+#echo "testing BNN quantizer..."
+#python3 BNN_quantizer_cifar10.py
+
+rm -rf ./checkpoints/*

test/tuner_test.py

-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT license.
-
-import sys
-import os.path as osp
-import subprocess
-import sys
-import time
-import traceback
-
-from utils import get_yml_content, dump_yml_content, setup_experiment, get_nni_log_path, is_experiment_done
-from utils import GREEN, RED, CLEAR, EXPERIMENT_URL
-
-TUNER_LIST = ['GridSearch', 'BatchTuner', 'TPE', 'Random', 'Anneal', 'Evolution']
-ASSESSOR_LIST = ['Medianstop']
-
-
-def get_config_file_path():
-    if sys.platform == 'win32':
-        config_file = osp.join('tuner_test', 'local_win32.yml')
-    else:
-        config_file = osp.join('tuner_test', 'local.yml')
-    return config_file
-
-def switch(dispatch_type, dispatch_name):
-    '''Change dispatch in config.yml'''
-    config_path = get_config_file_path()
-    experiment_config = get_yml_content(config_path)
-    if dispatch_name in ['GridSearch', 'BatchTuner', 'Random']:
-        experiment_config[dispatch_type.lower()] = {
-            'builtin' + dispatch_type + 'Name': dispatch_name
-        }
-    else:
-        experiment_config[dispatch_type.lower()] = {
-            'builtin' + dispatch_type + 'Name': dispatch_name,
-            'classArgs': {
-                'optimize_mode': 'maximize'
-            }
-        }
-    if dispatch_name == 'BatchTuner':
-        experiment_config['searchSpacePath'] = 'batchtuner_search_space.json'
-    else:
-        experiment_config['searchSpacePath'] = 'search_space.json'
-    dump_yml_content(config_path, experiment_config)
-
-def test_builtin_dispatcher(dispatch_type, dispatch_name):
-    '''test a dispatcher whose type is dispatch_type and name is dispatch_name'''
-    switch(dispatch_type, dispatch_name)
-
-    print('Testing %s...' % dispatch_name)
-    proc = subprocess.run(['nnictl', 'create', '--config', get_config_file_path()])
-    assert proc.returncode == 0, '`nnictl create` failed with code %d' % proc.returncode
-
-    nnimanager_log_path = get_nni_log_path(EXPERIMENT_URL)
-
-    for _ in range(20):
-        time.sleep(3)
-        # check if experiment is done
-        experiment_status = is_experiment_done(nnimanager_log_path)
-        if experiment_status:
-            break
-
-    assert experiment_status, 'Failed to finish in 1 min'
-
-def run(dispatch_type):
-    '''test all dispatchers whose type is dispatch_type'''
-    assert dispatch_type in ['Tuner', 'Assessor'], 'Unsupported dispatcher type: %s' % (dispatch_type)
-    dipsatcher_list = TUNER_LIST if dispatch_type == 'Tuner' else ASSESSOR_LIST
-    for dispatcher_name in dipsatcher_list:
-        try:
-            # Sleep here to make sure previous stopped exp has enough time to exit to avoid port conflict
-            time.sleep(6)
-            test_builtin_dispatcher(dispatch_type, dispatcher_name)
-            print(GREEN + 'Test %s %s: TEST PASS' % (dispatcher_name, dispatch_type) + CLEAR)
-        except Exception as error:
-            print(RED + 'Test %s %s: TEST FAIL' % (dispatcher_name, dispatch_type) + CLEAR)
-            print('%r' % error)
-            traceback.print_exc()
-            raise error
-        finally:
-            subprocess.run(['nnictl', 'stop'])
-
-if __name__ == '__main__':
-    installed = (sys.argv[-1] != '--preinstall')
-    setup_experiment(installed)
-
-    run('Tuner')
-    run('Assessor')

test/tuner_test/batchtuner_search_space.json

-{
-    "combine_params":
-    {
-        "_type" : "choice",
-        "_value" : [{"x": 1},
-                    {"x": 100}]
-    }
-}

test/tuner_test/local.yml

-assessor:
-  builtinAssessorName: Medianstop
-  classArgs:
-    optimize_mode: maximize
-authorName: nni
-experimentName: test_sdk
-maxExecDuration: 1h
-maxTrialNum: 2
-searchSpacePath: search_space.json
-trainingServicePlatform: local
-trial:
-  codeDir: .
-  command: python3 naive_trial.py
-  gpuNum: 0
-trialConcurrency: 2
-tuner:
-  builtinTunerName: Evolution
-  classArgs:
-    optimize_mode: maximize
-useAnnotation: false

test/tuner_test/local_win32.yml

-assessor:
-  builtinAssessorName: Medianstop
-  classArgs:
-    optimize_mode: maximize
-authorName: nni
-experimentName: test_sdk
-maxExecDuration: 1h
-maxTrialNum: 2
-searchSpacePath: search_space.json
-trainingServicePlatform: local
-trial:
-  codeDir: .
-  command: python naive_trial.py
-  gpuNum: 0
-trialConcurrency: 2
-tuner:
-  builtinTunerName: Evolution
-  classArgs:
-    optimize_mode: maximize
-useAnnotation: false

test/tuner_test/naive_trial.py

-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT license.
-
-import nni
-
-params = nni.get_next_parameter()
-print('params:', params)
-x = params['x']
-
-nni.report_final_result(x)