add it case: user installed builtin tuner (#2859)

Co-authored-by: Ning Shang <nishang@microsoft.com>

test/config/customized_tuners/demotuner-sklearn-classification.yml

+authorName: nni
+experimentName: default_test
+maxExecDuration: 5m
+maxTrialNum: 4
+trialConcurrency: 2
+searchSpacePath: ../../../examples/trials/sklearn/classification/search_space.json
+
+tuner:
+  builtinTunerName: demotuner
+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
+trial:
+  codeDir: ../../../examples/trials/sklearn/classification
+  command: python3 main.py
+  gpuNum: 0
+
+useAnnotation: false
+multiPhase: false
+multiThread: false
+
+trainingServicePlatform: local

test/config/integration_tests.yml

@@ -242,3 +242,9 @@ testCases:
 
 - name: tuner-metis
   configFile: test/config/tuners/metis.yml
+
+#########################################################################
+# nni customized-tuners test
+#########################################################################
+- name: customized-tuners-demotuner
+  configFile: test/config/customized_tuners/demotuner-sklearn-classification.yml

test/pipelines/pipelines-it-frameworkcontroller.yml

@@ -32,6 +32,13 @@ jobs:
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
     displayName: 'Install dependencies for integration tests in frameworkcontroller mode'
+  
+  - script: |
+      set -e
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      PATH=$HOME/.local/bin:$PATH nnictl package install ./
+    displayName: 'Install demotuner for customized_tuner test'
 
   - script: |
       set -e

test/pipelines/pipelines-it-kubeflow.yml

@@ -32,6 +32,13 @@ jobs:
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
     displayName: 'Install dependencies for integration tests in Kubeflow mode'
 
+  - script: |
+      set -e
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      PATH=$HOME/.local/bin:$PATH nnictl package install ./
+    displayName: 'Install demotuner for customized_tuner test'
+
   - script: |
       set -e
       if [ $(build_docker_img) = 'true' ]

test/pipelines/pipelines-it-local-windows.yml

@@ -13,6 +13,11 @@ jobs:
       python -m pip install tensorflow-gpu==1.15.2 tensorflow-estimator==1.15.1 --force --user
       nnictl package install --name=PPOTuner
     displayName: 'Install dependencies for integration tests'
+  - script: |
+      cd examples/tuners/customized_tuner
+      python setup.py develop --user
+      nnictl package install ./
+    displayName: 'Install demotuner for customized_tuner test'
   - script: |
       cd test
       powershell.exe -file scripts/unittest.ps1

test/pipelines/pipelines-it-local.yml

@@ -21,6 +21,12 @@ jobs:
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=PPOTuner
     displayName: 'Install dependencies for integration tests'
+  - script: |
+      set -e
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      PATH=$HOME/.local/bin:$PATH nnictl package install ./
+    displayName: 'Install demotuner for customized_tuner test'
   - script: |
       cd test
       source scripts/unittest.sh

test/pipelines/pipelines-it-pai-windows.yml

@@ -64,6 +64,12 @@ jobs:
       set PATH=$(ENV_PATH)
       python -m pip install scikit-learn==0.23.2 --user
     displayName: 'Install dependencies for integration tests'
+  - script: |
+      set PATH=$(ENV_PATH)
+      cd examples/tuners/customized_tuner
+      python setup.py develop --user
+      nnictl package install ./
+    displayName: 'Install demotuner for customized_tuner test'
   - script: |
       cd test
       set PATH=$(ENV_PATH)

test/pipelines/pipelines-it-pai.yml

@@ -33,6 +33,13 @@ jobs:
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
     displayName: 'Install dependencies for integration tests in PAI mode'
 
+  - script: |
+      set -e
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      PATH=$HOME/.local/bin:$PATH nnictl package install ./
+    displayName: 'Install demotuner for customized_tuner test'
+
   - script: |
       set -e
       if [ $(build_docker_img) = 'true' ]

test/pipelines/pipelines-it-remote-linux-to-linux.yml

@@ -21,6 +21,12 @@ jobs:
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
     displayName: 'Install dependencies for integration tests in remote mode'
+  - script: |
+      set -e
+      cd examples/tuners/customized_tuner
+      python3 setup.py develop --user
+      PATH=$HOME/.local/bin:$PATH nnictl package install ./
+    displayName: 'Install demotuner for customized_tuner test'
   - task: CopyFilesOverSSH@0
     inputs:
       sshEndpoint: $(end_point)

test/pipelines/pipelines-it-remote-linux-to-windows.yml

@@ -32,6 +32,12 @@ jobs:
           PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
           PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
         displayName: "Install dependencies for integration tests in remote mode"
+      - script: |
+          set -e
+          cd examples/tuners/customized_tuner
+          python3 setup.py develop --user
+          PATH=$HOME/.local/bin:$PATH nnictl package install ./
+        displayName: 'Install demotuner for customized_tuner test'
       - script: |
           set -e
           cd test

test/pipelines/pipelines-it-remote-windows-to-linux.yml

@@ -18,6 +18,11 @@ jobs:
   - script: |
       python -m pip install scikit-learn==0.23.2 --user
     displayName: 'Install dependencies for integration tests'
+  - script: |
+      cd examples/tuners/customized_tuner
+      python setup.py develop --user
+      nnictl package install ./
+    displayName: 'Install demotuner for customized_tuner test'
   - task: SSH@0
     inputs:
       sshEndpoint: $(end_point)