Refactor integration tests (#2190)

test/config/naive_trial/search_space.json

+{
+    "k":
+    {
+        "_type" : "randint",
+        "_value" : [0, 4]
+    },
+    "d":
+    {
+        "_type" : "choice",
+        "_value" : [-1, 1]
+    },
+    "n":
+    {
+        "_type" : "uniform",
+        "_value" : [0, 0.2]
+    }
+}

test/tuner_test/search_space.json → test/config/naive_trial/search_space_choices.json

 {
-    "x":
+    "p":
     {
         "_type" : "choice",
-        "_value" : [1, 100]
+        "_value" : [1, 2, 3, 4]
     }
 }

test/config/naive_trial/trial.py

+import random
+import time
+import math
+import nni
+
+curve_func = {
+    0: lambda x: x,
+    1: lambda x: x * x,
+    2: lambda x: math.pow(x, 0.5),
+    3: lambda x: math.tanh(x)
+}
+
+if __name__ == '__main__':
+    print('trial start')
+
+    params = nni.get_next_parameter()
+    print('params:', params)
+    epochs = 20
+
+    for i in range(epochs):
+        v = curve_func[params['k']](i / epochs)
+        v += v * (random.random() * params['n'])
+        v *= params['d']
+        nni.report_intermediate_result(v)
+
+        if i % 5 == 0:
+            time.sleep(1)
+    nni.report_final_result(v)
+    print('trial done')

test/config/naive_trial/trial_choices.py

+import random
+import nni
+
+if __name__ == '__main__':
+    print('trial start')
+
+    params = nni.get_next_parameter()
+    print('params:', params)
+
+    nni.report_intermediate_result(random.random())
+    nni.report_final_result(random.random())
+
+    print('trial done')

test/config/pr_tests.yml

+
+defaultTestCaseConfig:
+  launchCommand: nnictl create --config $configFile
+  stopCommand: nnictl stop
+  experimentStatusCheck: True
+  platform: linux darwin win32
+
+testCases:
+
+#########################################################################
+# naive test
+#########################################################################
+
+- name: naive-test
+  configFile: test/config/naive_test/local.yml
+  launchCommand: python3 nni_test/nnitest/naive_test.py --config $configFile
+  experimentStatusCheck: False
+  stopCommand:
+  platform: linux darwin
+
+#########################################################################
+# nni features test
+#########################################################################
+- name: metrics-float
+  configFile: test/config/metrics_test/config.yml
+  config:
+    maxTrialNum: 1
+    trialConcurrency: 1
+  validator:
+    class: MetricsValidator
+    kwargs:
+      expected_result_file: expected_metrics.json
+
+# to be enabled
+#- name: metrics-dict
+#  configFile: test/config/metrics_test/config_dict_metrics.yml
+#  config:
+#    maxTrialNum: 1
+#    trialConcurrency: 1
+#  validator:
+#    class: MetricsValidator
+#    kwargs:
+#      expected_result_file: expected_metrics_dict.json
+
+- name: nnicli
+  configFile: test/config/examples/sklearn-regression.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+  launchCommand: python3 -c 'import nnicli as nc; nc.start_nni("$configFile")'
+  stopCommand: python3 -c 'import nnicli as nc; nc.stop_nni()'
+  validator:
+    class: NnicliValidator
+  platform: linux darwin
+
+- name: multi-thread
+  configFile: test/config/multi_thread/config.yml
+
+- name: multi-phase-batch
+  configFile: test/config/multi_phase/batch.yml
+  config:
+    # for batch tuner, maxTrialNum can not exceed length of search space
+    maxTrialNum: 2
+    trialConcurrency: 2
+
+#########################################################################
+# nni assessor test
+#########################################################################
+- name: assessor-curvefitting
+  configFile: test/config/assessors/curvefitting.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+
+- name: assessor-medianstop
+  configFile: test/config/assessors/medianstop.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+
+#########################################################################
+# nni tuners test
+#########################################################################
+- name: tuner-annel
+  configFile: test/config/tuners/anneal.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+    useAnnotation: False
+    searchSpacePath: ../naive_trial/search_space.json
+    trial:
+      codeDir: ../naive_trial
+      command: python3 trial.py
+
+- name: tuner-evolution
+  configFile: test/config/tuners/evolution.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+    useAnnotation: False
+    searchSpacePath: ../naive_trial/search_space.json
+    trial:
+      codeDir: ../naive_trial
+      command: python3 trial.py
+
+- name: tuner-random
+  configFile: test/config/tuners/random.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+    useAnnotation: False
+    searchSpacePath: ../naive_trial/search_space.json
+    trial:
+      codeDir: ../naive_trial
+      command: python3 trial.py
+
+- name: tuner-tpe
+  configFile: test/config/tuners/tpe.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+    useAnnotation: False
+    searchSpacePath: ../naive_trial/search_space.json
+    trial:
+      codeDir: ../naive_trial
+      command: python3 trial.py
+
+- name: tuner-batch
+  configFile: test/config/tuners/batch.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+    useAnnotation: False
+    searchSpacePath: ../naive_trial/search_space_choices.json
+    trial:
+      codeDir: ../naive_trial
+      command: python3 trial_choices.py
+
+- name: tuner-gp
+  configFile: test/config/tuners/gp.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+    useAnnotation: False
+    searchSpacePath: ../naive_trial/search_space.json
+    trial:
+      codeDir: ../naive_trial
+      command: python3 trial.py
+
+- name: tuner-grid
+  configFile: test/config/tuners/gridsearch.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+    useAnnotation: False
+    searchSpacePath: ../naive_trial/search_space_choices.json
+    trial:
+      codeDir: ../naive_trial
+      command: python3 trial_choices.py

test/training_service.yml → test/config/training_service.yml

@@ -4,6 +4,10 @@ all:
 kubeflow:
   maxExecDuration: 15m
   nniManagerIp:
+  # use a small trial number to make IT faster
+  maxTrialNum: 2
+  trialConcurrency: 2
+
   kubeflowConfig:
     operator: tf-operator
     apiVersion: v1alpha2
@@ -27,6 +31,9 @@ kubeflow:
 frameworkcontroller:
   maxExecDuration: 15m
   nniManagerIp:
+  # use a small trial number to make IT faster
+  maxTrialNum: 2
+  trialConcurrency: 2
   frameworkcontrollerConfig:
     serviceAccountName: frameworkbarrier
     storage: azureStorage