support tf2 NAS with non-weight-sharing mode (#2541)

test/config/examples/classic-nas-tf2.yml

+authorName: nni
+experimentName: default_test
+maxExecDuration: 10m
+maxTrialNum: 1
+trialConcurrency: 1
+searchSpacePath: nni-nas-search-space-tf2.json
+
+tuner:
+  builtinTunerName: PPOTuner
+  classArgs:
+    optimize_mode: maximize
+trial:
+  command: python3 train.py --epochs 1
+  codeDir: ../../../examples/nas/classic_nas-tf
+  gpuNum: 0
+
+useAnnotation: false
+multiPhase: false
+multiThread: false
+
+trainingServicePlatform: local
\ No newline at end of file

test/config/integration_tests.yml

@@ -72,6 +72,23 @@ testCases:
 - name: nested-ss
   configFile: test/config/examples/mnist-nested-search-space.yml
 
+- name: classic-nas-gen-ss
+  configFile: test/config/examples/classic-nas-pytorch.yml
+  launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/classic_nas --file=config/examples/nni-nas-search-space.json
+  stopCommand:
+  experimentStatusCheck: False
+
+- name: classic-nas-install-ppo
+  configFile: test/config/examples/classic-nas-pytorch.yml
+  launchCommand: nnictl package install --name=PPOTuner
+  stopCommand:
+  experimentStatusCheck: False
+
+- name: classic-nas-pytorch
+  configFile: test/config/examples/classic-nas-pytorch.yml
+  # remove search space file
+  stopCommand: nnictl stop
+  onExitCommand: python3 -c 'import os; os.remove("config/examples/nni-nas-search-space.json")'
 
 #########################################################################
 # nni features test

test/config/integration_tests_tf2.yml

@@ -51,6 +51,24 @@ testCases:
       command: python3 main.py --epochs 1 --batches 1
       gpuNum: 0
 
+- name: classic-nas-gen-ss
+  configFile: test/config/examples/classic-nas-tf2.yml
+  launchCommand: nnictl ss_gen --trial_command="python3 train.py --epochs 1" --trial_dir=../examples/nas/classic_nas-tf --file=config/examples/nni-nas-search-space-tf2.json
+  stopCommand:
+  experimentStatusCheck: False
+
+- name: classic-nas-install-ppo
+  configFile: test/config/examples/classic-nas-tf2.yml
+  launchCommand: nnictl package install --name=PPOTuner
+  stopCommand:
+  experimentStatusCheck: False
+
+- name: classic-nas-tensorflow2
+  configFile: test/config/examples/classic-nas-tf2.yml
+  # remove search space file
+  stopCommand: nnictl stop
+  onExitCommand: python3 -c 'import os; os.remove("config/examples/nni-nas-search-space-tf2.json")'
+
 #########################################################################
 # nni features test
 #########################################################################

test/nni_test/nnitest/run_tests.py

@@ -76,6 +76,10 @@ def run_test_case(test_case_config, it_config, args):
         print('Stop command:', stop_command, flush=True)
         if stop_command:
             subprocess.run(shlex.split(stop_command))
+        exit_command = get_command(test_case_config, 'onExitCommand')
+        print('Exit command:', exit_command, flush=True)
+        if exit_command:
+            subprocess.run(shlex.split(exit_command), check=True)
         # remove tmp config file
         if os.path.exists(new_config_file):
             os.remove(new_config_file)

test/scripts/nas.sh

@@ -6,7 +6,7 @@ echo ""
 echo "===========================Testing: NAS==========================="
 EXAMPLE_DIR=${CWD}/../examples/nas
 
-echo "testing classic nas..."
+echo "testing nnictl ss_gen (classic nas)..."
 cd $EXAMPLE_DIR/classic_nas
 SEARCH_SPACE_JSON=nni_auto_gen_search_space.json
 if [ -f $SEARCH_SPACE_JSON ]; then