Fix it pipelines (#2605)

test/config/integration_tests.yml

@@ -4,6 +4,7 @@ defaultTestCaseConfig:
   stopCommand: nnictl stop
   experimentStatusCheck: True
   platform: linux darwin win32
+  trainingService: all
 
 testCases:
 #######################################################################
@@ -77,18 +78,14 @@ testCases:
   launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/classic_nas --file=config/examples/nni-nas-search-space.json
   stopCommand:
   experimentStatusCheck: False
-
-- name: classic-nas-install-ppo
-  configFile: test/config/examples/classic-nas-pytorch.yml
-  launchCommand: nnictl package install --name=PPOTuner
-  stopCommand:
-  experimentStatusCheck: False
+  trainingService: local
 
 - name: classic-nas-pytorch
   configFile: test/config/examples/classic-nas-pytorch.yml
   # remove search space file
   stopCommand: nnictl stop
   onExitCommand: python3 -c 'import os; os.remove("config/examples/nni-nas-search-space.json")'
+  trainingService: local
 
 #########################################################################
 # nni features test

test/config/integration_tests_tf2.yml

@@ -4,6 +4,7 @@ defaultTestCaseConfig:
   stopCommand: nnictl stop
   experimentStatusCheck: True
   platform: linux darwin win32
+  trainingService: all
 
 testCases:
 #######################################################################
@@ -56,18 +57,14 @@ testCases:
   launchCommand: nnictl ss_gen --trial_command="python3 train.py --epochs 1" --trial_dir=../examples/nas/classic_nas-tf --file=config/examples/nni-nas-search-space-tf2.json
   stopCommand:
   experimentStatusCheck: False
-
-- name: classic-nas-install-ppo
-  configFile: test/config/examples/classic-nas-tf2.yml
-  launchCommand: nnictl package install --name=PPOTuner
-  stopCommand:
-  experimentStatusCheck: False
+  trainingService: local
 
 - name: classic-nas-tensorflow2
   configFile: test/config/examples/classic-nas-tf2.yml
   # remove search space file
   stopCommand: nnictl stop
   onExitCommand: python3 -c 'import os; os.remove("config/examples/nni-nas-search-space-tf2.json")'
+  trainingService: local
 
 #########################################################################
 # nni features test

test/config/pr_tests.yml

@@ -4,6 +4,7 @@ defaultTestCaseConfig:
   stopCommand: nnictl stop
   experimentStatusCheck: True
   platform: linux darwin win32
+  trainingService: all
 
 testCases:
 

test/nni_test/nnitest/run_tests.py

@@ -196,6 +196,15 @@ def case_included(name, cases):
 def match_platform(test_case_config):
     return sys.platform in test_case_config['platform'].split(' ')
 
+def match_training_service(test_case_config, cur_training_service):
+    case_ts = test_case_config['trainingService']
+    assert case_ts is not None
+    if case_ts == 'all':
+        return True
+    if cur_training_service in case_ts.split(' '):
+        return True
+    return False
+
 def run(args):
     it_config = get_yml_content(args.config)
 
@@ -217,6 +226,10 @@ def run(args):
             print('skipped {}, platform {} not match [{}]'.format(name, sys.platform, test_case_config['platform']))
             continue
 
+        if not match_training_service(test_case_config, args.ts):
+            print('skipped {}, training service {} not match [{}]'.format(name, args.ts, test_case_config['trainingService']))
+            continue
+
         wait_for_port_available(8080, 30)
         print('{}Testing: {}{}'.format(GREEN, name, CLEAR))
         begin_time = time.time()

test/pipelines/pipelines-it-local-tf2.yml

@@ -17,6 +17,7 @@ jobs:
       sudo apt-get install swig -y
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
+      PATH=$HOME/.local/bin:$PATH nnictl package install --name=PPOTuner
     displayName: 'Install dependencies for integration tests'
   - script: |
       cd test

test/pipelines/pipelines-it-local-windows.yml

@@ -11,6 +11,7 @@ jobs:
       python -m pip install keras==2.1.6 --user
       python -m pip install torchvision===0.4.1 torch===1.3.1 -f https://download.pytorch.org/whl/torch_stable.html --user
       python -m pip install tensorflow-gpu==1.15.2 tensorflow-estimator==1.15.1 --force --user
+      nnictl package install --name=PPOTuner
     displayName: 'Install dependencies for integration tests'
   - script: |
       cd test

test/pipelines/pipelines-it-local.yml

@@ -18,6 +18,7 @@ jobs:
       sudo apt-get install swig -y
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
       PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
+      PATH=$HOME/.local/bin:$PATH nnictl package install --name=PPOTuner
     displayName: 'Install dependencies for integration tests'
   - script: |
       cd test