Add pipeline for tfv2 (#2584)

azure-pipelines.yml

@@ -10,7 +10,6 @@ jobs:
   - script: |
       set -e
       python3 -m pip install --upgrade pip setuptools --user
-      python3 -m pip install pylint==2.3.1 astroid==2.2.5 --user
       python3 -m pip install coverage --user
       echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
     displayName: 'Install python tools'
@@ -28,29 +27,14 @@ jobs:
   - script: |
       set -e
       python3 -m pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html --user
-      python3 -m pip install tensorflow==1.15.2 --user
-      python3 -m pip install keras==2.1.6 --user
+      python3 -m pip install tensorflow==2.2.0 --user
+      python3 -m pip install keras==2.4.2 --user
       python3 -m pip install gym onnx --user
       python3 -m pip install sphinx==1.8.3 sphinx-argparse==0.2.5 sphinx-markdown-tables==0.0.9 sphinx-rtd-theme==0.4.2 sphinxcontrib-websupport==1.1.0 recommonmark==0.5.0 --user
       sudo apt-get install swig -y
       nnictl package install --name=SMAC
       nnictl package install --name=BOHB
     displayName: 'Install dependencies'
-  - script: |
-      set -e
-      python3 -m pylint --rcfile pylintrc nni_annotation
-      python3 -m pylint --rcfile pylintrc nni_cmd
-      python3 -m pylint --rcfile pylintrc nni_gpu_tool
-      python3 -m pylint --rcfile pylintrc nni_trial_tool
-      python3 -m pylint --rcfile pylintrc nni
-      python3 -m pylint --rcfile pylintrc nnicli
-    displayName: 'Run pylint'
-  - script: |
-      set -e
-      python3 -m pip install flake8 --user
-      EXCLUDES=./src/nni_manager/,./src/webui,./tools/nni_annotation/testcase/,./examples/trials/mnist-nas/*/mnist*.py,./examples/trials/nas_cifar10/src/cifar10/general_child.py
-      python3 -m flake8 . --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics
-    displayName: 'Run flake8 tests to find Python syntax errors and undefined names'
   - script: |
       cd test
       source scripts/unittest.sh
@@ -64,7 +48,7 @@ jobs:
       sphinx-build -M html . _build -W
     displayName: 'Sphinx Documentation Build check'
 
-- job: 'ubuntu_1604_python35_legacy_torch'
+- job: 'ubuntu_1604_python35_legacy_torch_tf'
   pool:
     vmImage: 'Ubuntu 16.04'
 
@@ -72,6 +56,7 @@ jobs:
   - script: |
       set -e
       python3 -m pip install --upgrade pip setuptools --user
+      python3 -m pip install pylint==2.3.1 astroid==2.2.5 --user
       python3 -m pip install coverage --user
       echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
     displayName: 'Install python tools'
@@ -88,6 +73,21 @@ jobs:
       nnictl package install --name=SMAC
       nnictl package install --name=BOHB
     displayName: 'Install dependencies'
+  - script: |
+      set -e
+      python3 -m pylint --rcfile pylintrc nni_annotation
+      python3 -m pylint --rcfile pylintrc nni_cmd
+      python3 -m pylint --rcfile pylintrc nni_gpu_tool
+      python3 -m pylint --rcfile pylintrc nni_trial_tool
+      python3 -m pylint --rcfile pylintrc nni
+      python3 -m pylint --rcfile pylintrc nnicli
+    displayName: 'Run pylint'
+  - script: |
+      set -e
+      python3 -m pip install flake8 --user
+      EXCLUDES=./src/nni_manager/,./src/webui,./tools/nni_annotation/testcase/,./examples/trials/mnist-nas/*/mnist*.py,./examples/trials/nas_cifar10/src/cifar10/general_child.py
+      python3 -m flake8 . --count --exclude=$EXCLUDES --select=E9,F63,F72,F82 --show-source --statistics
+    displayName: 'Run flake8 tests to find Python syntax errors and undefined names'
   - script: |
       cd test
       source scripts/unittest.sh

src/sdk/pynni/tests/test_compressor.py

@@ -143,7 +143,7 @@ class CompressorTestCase(TestCase):
 
     @tf2
     def test_tf_fpgm_pruner(self):
-        w = np.array([np.ones((5, 5, 5)) * (i+1) for i in range(10)]).astype(np.float32)
+        w = np.array([np.ones((5, 3, 3)) * (i+1) for i in range(10)]).astype(np.float32)
         model = get_tf_model()
         config_list = [{'sparsity': 0.2, 'op_types': ['Conv2D']}]
 

test/config/integration_tests_tf2.yml

+
+defaultTestCaseConfig:
+  launchCommand: nnictl create --config $configFile --debug
+  stopCommand: nnictl stop
+  experimentStatusCheck: True
+  platform: linux darwin win32
+
+testCases:
+#######################################################################
+# nni examples test
+#######################################################################
+- name: sklearn-classification
+  # test case config yml file relative to nni source code directory
+  configFile: test/config/examples/sklearn-classification.yml
+
+  # test case specific config, the content of configFile will be overrided
+  # by config section
+  config:
+
+  # validator is called after experiment is done
+  # validator class needs to be implemented in nni_test/nnitest/validators.py
+  validator:
+
+  # launch command, default launch command is 'nnictl create --config $configFile'
+  launchCommand: nnictl create --config $configFile --debug
+
+  # stop command, default stop command is 'nnictl stop', empty means no stop command
+  stopCommand: nnictl stop
+
+  # set experiment ID into variable, variable name should start with $, such as $expId
+  setExperimentIdtoVar: $expId
+
+  # check status of experiment before calling validator
+  experimentStatusCheck: True
+
+- name: sklearn-regression
+  configFile: test/config/examples/sklearn-regression.yml
+
+- name: mnist-pytorch
+  configFile: test/config/examples/mnist-pytorch.yml
+
+- name: cifar10-pytorch
+  configFile: test/config/examples/cifar10-pytorch.yml
+  config:
+    # this example downloads large pretrained model weights
+    # test 1 trial to save time
+    maxExecDuration: 10m
+    maxTrialNum: 1
+    trialConcurrency: 1
+    trial:
+      command: python3 main.py --epochs 1 --batches 1
+      gpuNum: 0
+
+#########################################################################
+# nni features test
+#########################################################################
+- name: metrics-float
+  configFile: test/config/metrics_test/config.yml
+  config:
+    maxTrialNum: 1
+    trialConcurrency: 1
+  validator:
+    class: MetricsValidator
+    kwargs:
+      expected_result_file: expected_metrics.json
+
+- name: export-float
+  configFile: test/config/metrics_test/config.yml
+  config:
+    maxTrialNum: 1
+    trialConcurrency: 1
+  validator:
+    class: ExportValidator
+
+- name: metrics-dict
+  configFile: test/config/metrics_test/config_dict_metrics.yml
+  config:
+    maxTrialNum: 1
+    trialConcurrency: 1
+  validator:
+    class: MetricsValidator
+    kwargs:
+      expected_result_file: expected_metrics_dict.json
+
+- name: export-dict
+  configFile: test/config/metrics_test/config_dict_metrics.yml
+  config:
+    maxTrialNum: 1
+    trialConcurrency: 1
+  validator:
+    class: ExportValidator
+
+- name: nnicli
+  configFile: test/config/examples/sklearn-regression.yml
+  config:
+    maxTrialNum: 4
+    trialConcurrency: 4
+  launchCommand: python3 -c 'import nnicli as nc; nc.start_nni("$configFile")'
+  stopCommand: python3 -c 'import nnicli as nc; nc.stop_nni()'
+  validator:
+    class: NnicliValidator
+  platform: linux darwin
+
+- name: foreground
+  configFile: test/config/examples/sklearn-regression.yml
+  launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45
+  stopCommand:
+  experimentStatusCheck: False
+  platform: linux darwin
+
+# Experiment resume test part 1
+- name: nnictl-resume-1
+  configFile: test/config/examples/sklearn-regression.yml
+  setExperimentIdtoVar: $resumeExpId
+
+# Experiment resume test part 2
+- name: nnictl-resume-2
+  configFile: test/config/examples/sklearn-regression.yml
+  launchCommand: nnictl resume $resumeExpId
+
+# Experiment view test
+- name: nnictl-view
+  configFile: test/config/examples/sklearn-regression.yml
+  launchCommand: nnictl view $resumeExpId
+  experimentStatusCheck: False
+
+- name: multi-thread
+  configFile: test/config/multi_thread/config.yml
+
+- name: multi-phase-batch
+  configFile: test/config/multi_phase/batch.yml
+  config:
+    # for batch tuner, maxTrialNum can not exceed length of search space
+    maxTrialNum: 2
+    trialConcurrency: 2
+
+- name: multi-phase-evolution
+  configFile: test/config/multi_phase/evolution.yml
+
+- name: multi-phase-grid
+  configFile: test/config/multi_phase/grid.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+
+- name: multi-phase-metis
+  configFile: test/config/multi_phase/metis.yml
+
+- name: multi-phase-tpe
+  configFile: test/config/multi_phase/tpe.yml
+
+#########################################################################
+# nni assessor test
+#########################################################################
+- name: assessor-curvefitting
+  configFile: test/config/assessors/curvefitting.yml
+
+- name: assessor-medianstop
+  configFile: test/config/assessors/medianstop.yml

test/pipelines/pipelines-it-local-tf2.yml

+jobs:
+- job: 'integration_test_local_ubuntu'
+  timeoutInMinutes: 120
+
+  steps:
+  - script: python3 -m pip install --upgrade pip setuptools --user
+    displayName: 'Install python tools'
+  - script: |
+      source install.sh
+    displayName: 'Install nni toolkit via source code'
+  - script: |
+      set -e
+      python3 -m pip install scikit-learn==0.20.0 --user
+      python3 -m pip install torch==1.3.1 torchvision==0.4.1 -f https://download.pytorch.org/whl/torch_stable.html --user
+      python3 -m pip install tensorflow-gpu==2.2.0 --user
+      python3 -m pip install keras==2.4.2 --user
+      sudo apt-get install swig -y
+      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
+      PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
+    displayName: 'Install dependencies for integration tests'
+  - script: |
+      cd test
+      source scripts/unittest.sh
+    displayName: 'Unit test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests_tf2.yml --ts local
+    displayName: 'Integration test'
+  - script: |
+      cd test
+      PATH=$HOME/.local/bin:$PATH source scripts/nas.sh
+    displayName: 'NAS test'
+  - script: |
+      cd test
+      source scripts/model_compression.sh
+    displayName: 'Model compression test'