Refactor integration tests (#2190)

azure-pipelines.yml

@@ -52,24 +52,12 @@ jobs:
     displayName: 'Run flake8 tests to find Python syntax errors and undefined names'
   - script: |
       cd test
-      source unittest.sh
+      source scripts/unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      python3 naive_test.py
-    displayName: 'Naive test'
-  - script: |
-      cd test
-      python3 tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
-  - script: |
-      cd test
-      python3 metrics_test.py
-    displayName: 'Trial job metrics test'
-  - script: |
-      cd test
-      python3 cli_test.py
-    displayName: 'nnicli test'
+      python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+    displayName: 'Simple test'
   - script: |
       cd docs/en_US/
       sphinx-build -M html . _build -W
@@ -101,20 +89,12 @@ jobs:
     displayName: 'Install dependencies'
   - script: |
       cd test
-      source unittest.sh
+      source scripts/unittest.sh
     displayName: 'Unit test'
   - script: |
       cd test
-      python3 naive_test.py
-    displayName: 'Naive test'
-  - script: |
-      cd test
-      python3 tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
-  - script: |
-      cd test
-      python3 cli_test.py
-    displayName: 'nnicli test'
+      python3 nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+    displayName: 'Simple test'
 
 - job: 'basic_test_pr_Windows'
   pool:
@@ -137,13 +117,9 @@ jobs:
     displayName: 'Install dependencies'
   - script: |
       cd test
-      powershell.exe -file unittest.ps1
+      powershell.exe -file scripts/unittest.ps1
     displayName: 'unit test'
   - script: |
       cd test
-      python tuner_test.py
-    displayName: 'Built-in tuners / assessors tests'
-  - script: |
-      cd test
-      PATH=$HOME/.local/bin:$PATH python3 cli_test.py
-    displayName: 'nnicli test'
+      python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+    displayName: 'Simple test'

examples/model_compress/model_prune_torch.py

@@ -67,7 +67,7 @@ prune_config = {
         'dataset_name': 'cifar10',
         'model_name': 'vgg16',
         'pruner_class': ActivationMeanRankFilterPruner,
-        'configure_list': [{
+        'config_list': [{
             'sparsity': 0.5,
             'op_types': ['default'],
             'op_names': ['feature.0', 'feature.24', 'feature.27', 'feature.30', 'feature.34', 'feature.37']

test/cli_test.py

-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT license.
-
-import sys
-import time
-import traceback
-from utils import GREEN, RED, CLEAR, setup_experiment
-
-def test_nni_cli():
-    import nnicli as nc
-
-    config_file = 'config_test/examples/mnist-tfv1.test.yml'
-
-    try:
-        # Sleep here to make sure previous stopped exp has enough time to exit to avoid port conflict
-        time.sleep(6)
-        print(GREEN + 'Testing nnicli:' + config_file + CLEAR)
-        nc.start_nni(config_file)
-        time.sleep(3)
-        nc.set_endpoint('http://localhost:8080')
-        print(nc.version())
-        print(nc.get_job_statistics())
-        print(nc.get_experiment_status())
-        nc.list_trial_jobs()
-
-        print(GREEN + 'Test nnicli {}: TEST PASS'.format(config_file) + CLEAR)
-    except Exception as error:
-        print(RED + 'Test nnicli {}: TEST FAIL'.format(config_file) + CLEAR)
-        print('%r' % error)
-        traceback.print_exc()
-        raise error
-    finally:
-        nc.stop_nni()
-
-if __name__ == '__main__':
-    installed = (sys.argv[-1] != '--preinstall')
-    setup_experiment(installed)
-
-    test_nni_cli()

test/config_test/tuners/mnist-curvefitting.test.yml → test/config/assessors/curvefitting.yml

 authorName: nni
 experimentName: default_test
 maxExecDuration: 5m
-maxTrialNum: 2
-trialConcurrency: 1
-searchSpacePath: search_space.json
+maxTrialNum: 8
+trialConcurrency: 8
+searchSpacePath: ../naive_trial/search_space.json
 
 tuner:
   builtinTunerName: TPE
@@ -17,12 +17,12 @@ assessor:
       start_step: 6
       threshold: 0.95
 trial:
-  codeDir: ../../../examples/trials/mnist-tfv1
-  command: python3 mnist.py --batch_num 100
+  codeDir: ../naive_trial
+  command: python3 trial.py
   gpuNum: 0
 
 useAnnotation: false
 multiPhase: false
 multiThread: false
 
-trainingServicePlatform: local
\ No newline at end of file
+trainingServicePlatform: local

test/config_test/multi_phase/multi_phase_tpe.test.yml → test/config/assessors/medianstop.yml

@@ -2,21 +2,24 @@ authorName: nni
 experimentName: default_test
 maxExecDuration: 5m
 maxTrialNum: 8
-trialConcurrency: 4
-searchSpacePath: ./search_space.json
+trialConcurrency: 8
+searchSpacePath: ../naive_trial/search_space.json
 
 tuner:
   builtinTunerName: TPE
   classArgs:
     optimize_mode: maximize
 
+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
+
 trial:
-  codeDir: .
-  command: python3 multi_phase.py
+  codeDir: ../naive_trial
+  command: python3 trial.py
   gpuNum: 0
 
 useAnnotation: false
-multiPhase: true
-multiThread: false
 
 trainingServicePlatform: local

test/config_test/examples/mnist-annotation.test.yml → test/config/examples/mnist-annotation.yml

@@ -12,7 +12,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-annotation
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: true

test/config_test/examples/mnist-nested-search-space.test.yml → test/config/examples/mnist-nested-search-space.yml

@@ -14,7 +14,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-nested-search-space
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: false

test/config_test/examples/mnist-pytorch.test.yml → test/config/examples/mnist-pytorch.yml

@@ -13,7 +13,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-pytorch
-  command: python3 mnist.py  --epochs 2
+  command: python3 mnist.py  --epochs 1
   gpuNum: 0
 
 useAnnotation: false

test/config_test/examples/mnist-tfv1.test.yml → test/config/examples/mnist-tfv1.yml

@@ -13,7 +13,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-tfv1
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: false

test/config/integration_tests.yml

+
+defaultTestCaseConfig:
+  launchCommand: nnictl create --config $configFile
+  stopCommand: nnictl stop
+  experimentStatusCheck: True
+  platform: linux darwin win32
+
+testCases:
+#######################################################################
+# nni examples test
+#######################################################################
+- name: sklearn-classification
+  # test case config yml file relative to nni source code directory
+  configFile: test/config/examples/sklearn-classification.yml
+
+  # test case specific config, the content of configFile will be overrided
+  # by config section
+  config:
+
+  # validator is called after experiment is done
+  # validator class needs to be implemented in nni_test/nnitest/validators.py
+  validator:
+
+  # launch command, default launch command is 'nnictl create --config $configFile'
+  launchCommand: nnictl create --config $configFile
+
+  # stop command, default stop command is 'nnictl stop', empty means no stop command
+  stopCommand: nnictl stop
+
+  # set experiment ID into variable, variable name should start with $, such as $expId
+  setExperimentIdtoVar: $expId
+
+  # check status of experiment before calling validator
+  experimentStatusCheck: True
+
+- name: sklearn-regression
+  configFile: test/config/examples/sklearn-regression.yml
+
+- name: mnist-tfv1
+  configFile: test/config/examples/mnist-tfv1.yml
+
+- name: mnist-keras
+  configFile: test/config/examples/mnist-keras.yml
+
+- name: mnist-pytorch
+  configFile: test/config/examples/mnist-pytorch.yml
+
+- name: mnist-annotation
+  configFile: test/config/examples/mnist-annotation.yml
+
+- name: cifar10-pytorch
+  configFile: test/config/examples/cifar10-pytorch.yml
+  config:
+    # this example downloads large pretrained model weights
+    # test 1 trial to save time
+    maxExecDuration: 10m
+    maxTrialNum: 1
+    trialConcurrency: 1
+    trial:
+      command: python3 main.py --epochs 1 --batches 1
+      gpuNum: 0
+
+- name: nested-ss
+  configFile: test/config/examples/mnist-nested-search-space.yml
+
+
+#########################################################################
+# nni features test
+#########################################################################
+- name: metrics-float
+  configFile: test/config/metrics_test/config.yml
+  config:
+    maxTrialNum: 1
+    trialConcurrency: 1
+  validator:
+    class: MetricsValidator
+    kwargs:
+      expected_result_file: expected_metrics.json
+
+# to be enabled
+#- name: metrics-dict
+#  configFile: test/config/metrics_test/config_dict_metrics.yml
+#  config:
+#    maxTrialNum: 1
+#    trialConcurrency: 1
+#  validator:
+#    class: MetricsValidator
+#    kwargs:
+#      expected_result_file: expected_metrics_dict.json
+
+- name: nnicli
+  configFile: test/config/examples/sklearn-regression.yml
+  config:
+    maxTrialNum: 4
+    trialConcurrency: 4
+  launchCommand: python3 -c 'import nnicli as nc; nc.start_nni("$configFile")'
+  stopCommand: python3 -c 'import nnicli as nc; nc.stop_nni()'
+  validator:
+    class: NnicliValidator
+  platform: linux darwin
+
+- name: foreground
+  configFile: test/config/examples/sklearn-regression.yml
+  launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45
+  stopCommand:
+  experimentStatusCheck: False
+  platform: linux darwin
+
+# Experiment resume test part 1
+- name: nnictl-resume-1
+  configFile: test/config/examples/sklearn-regression.yml
+  setExperimentIdtoVar: $resumeExpId
+
+# Experiment resume test part 2
+- name: nnictl-resume-2
+  configFile: test/config/examples/sklearn-regression.yml
+  launchCommand: nnictl resume $resumeExpId
+
+# Experiment view test
+- name: nnictl-view
+  configFile: test/config/examples/sklearn-regression.yml
+  launchCommand: nnictl view $resumeExpId
+  experimentStatusCheck: False
+
+- name: multi-thread
+  configFile: test/config/multi_thread/config.yml
+
+- name: multi-phase-batch
+  configFile: test/config/multi_phase/batch.yml
+  config:
+    # for batch tuner, maxTrialNum can not exceed length of search space
+    maxTrialNum: 2
+    trialConcurrency: 2
+
+- name: multi-phase-evolution
+  configFile: test/config/multi_phase/evolution.yml
+
+- name: multi-phase-grid
+  configFile: test/config/multi_phase/grid.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+
+- name: multi-phase-metis
+  configFile: test/config/multi_phase/metis.yml
+
+- name: multi-phase-tpe
+  configFile: test/config/multi_phase/tpe.yml
+
+#########################################################################
+# nni assessor test
+#########################################################################
+- name: assessor-curvefitting
+  configFile: test/config/assessors/curvefitting.yml
+
+- name: assessor-medianstop
+  configFile: test/config/assessors/medianstop.yml
+
+#########################################################################
+# nni tuners test
+#########################################################################
+- name: tuner-annel
+  configFile: test/config/tuners/anneal.yml
+
+- name: tuner-evolution
+  configFile: test/config/tuners/evolution.yml
+
+- name: tuner-random
+  configFile: test/config/tuners/random.yml
+
+- name: tuner-smac
+  configFile: test/config/tuners/smac.yml
+  platform: linux darwin
+
+- name: tuner-tpe
+  configFile: test/config/tuners/tpe.yml
+
+- name: tuner-batch
+  configFile: test/config/tuners/batch.yml
+
+- name: tuner-bohb
+  configFile: test/config/tuners/bohb.yml
+  platform: linux darwin
+
+- name: tuner-gp
+  configFile: test/config/tuners/gp.yml
+
+- name: tuner-grid
+  configFile: test/config/tuners/gridsearch.yml
+
+- name: tuner-hyperband
+  configFile: test/config/tuners/hyperband.yml
+
+- name: tuner-metis
+  configFile: test/config/tuners/metis.yml

test/metrics_test/metrics_win32.test.yml → test/config/metrics_test/config_dict_metrics.yml

@@ -10,7 +10,7 @@ tuner:
 
 trial:
   codeDir: .
-  command: python trial.py
+  command: python3 trial.py --dict_metrics
   gpuNum: 0
 
 useAnnotation: false