Merge pull request #238 from microsoft/master

merge master

src/webui/src/components/Modals/LogDrawer.tsx

@@ -154,7 +154,7 @@ class LogDrawer extends React.Component<LogDrawerProps, LogDrawerState> {
         });
         Promise.all([dispatcherPromise, nniManagerPromise]).then(() => {
             this.setState({ isLoading: false });
-            this.timerId = window.setTimeout(this.refresh, 300);
+            this.timerId = window.setTimeout(this.refresh, 10000);
         });
     }
 

src/webui/src/static/function.ts

@@ -77,8 +77,10 @@ const getFinal = (final?: MetricDataRecord[]): FinalType | undefined => {
     if (final) {
         showDefault = parseMetrics(final[final.length - 1].data);
         if (typeof showDefault === 'number') {
-            showDefault = { default: showDefault };
-            return showDefault;
+            if(!isNaN(showDefault)){
+                showDefault = { default: showDefault };
+                return showDefault;
+            }
         } else if (isArrayType(showDefault)) {
             // not support final type
             return undefined;

src/webui/src/static/model/trial.ts

@@ -5,7 +5,7 @@ class Trial implements TableObj {
     private metricsInitialized: boolean = false;
     private infoField: TrialJobInfo | undefined;
     private intermediates: (MetricDataRecord | undefined)[] = [ ];
-    private final: MetricDataRecord | undefined;
+    public final: MetricDataRecord | undefined;
     private finalAcc: number | undefined;
 
     constructor(info?: TrialJobInfo, metrics?: MetricDataRecord[]) {
@@ -211,13 +211,21 @@ class Trial implements TableObj {
 
     public formatLatestAccuracy(): string {  // TODO: this should be private
         if (this.accuracy !== undefined) {
-            return `${formatAccuracy(this.accuracy)} (FINAL)`;
+            if (isNaN(this.accuracy)) {
+                return this.accuracy.toString();
+            } else {
+                return `${formatAccuracy(this.accuracy)} (FINAL)`;
+            }
         } else if (this.intermediates.length === 0) {
             return '--';
         } else {
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
             const latest = this.intermediates[this.intermediates.length - 1]!;
-            return `${formatAccuracy(metricAccuracy(latest))} (LATEST)`;
+            if (isNaN(metricAccuracy(latest))) {
+                return 'NaN';
+            } else {
+                return `${formatAccuracy(metricAccuracy(latest))} (LATEST)`;
+            }
         }
     }
 }

test/cli_test.py

-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT license.
-
-import sys
-import time
-import traceback
-from utils import GREEN, RED, CLEAR, setup_experiment
-
-def test_nni_cli():
-    import nnicli as nc
-
-    config_file = 'config_test/examples/mnist-tfv1.test.yml'
-
-    try:
-        # Sleep here to make sure previous stopped exp has enough time to exit to avoid port conflict
-        time.sleep(6)
-        print(GREEN + 'Testing nnicli:' + config_file + CLEAR)
-        nc.start_nni(config_file)
-        time.sleep(3)
-        nc.set_endpoint('http://localhost:8080')
-        print(nc.version())
-        print(nc.get_job_statistics())
-        print(nc.get_experiment_status())
-        nc.list_trial_jobs()
-
-        print(GREEN + 'Test nnicli {}: TEST PASS'.format(config_file) + CLEAR)
-    except Exception as error:
-        print(RED + 'Test nnicli {}: TEST FAIL'.format(config_file) + CLEAR)
-        print('%r' % error)
-        traceback.print_exc()
-        raise error
-    finally:
-        nc.stop_nni()
-
-if __name__ == '__main__':
-    installed = (sys.argv[-1] != '--preinstall')
-    setup_experiment(installed)
-
-    test_nni_cli()

test/config_test/tuners/mnist-curvefitting.test.yml → test/config/assessors/curvefitting.yml

 authorName: nni
 experimentName: default_test
 maxExecDuration: 5m
-maxTrialNum: 2
-trialConcurrency: 1
-searchSpacePath: search_space.json
+maxTrialNum: 8
+trialConcurrency: 8
+searchSpacePath: ../naive_trial/search_space.json
 
 tuner:
   builtinTunerName: TPE
@@ -17,12 +17,12 @@ assessor:
       start_step: 6
       threshold: 0.95
 trial:
-  codeDir: ../../../examples/trials/mnist-tfv1
-  command: python3 mnist.py --batch_num 100
+  codeDir: ../naive_trial
+  command: python3 trial.py
   gpuNum: 0
 
 useAnnotation: false
 multiPhase: false
 multiThread: false
 
-trainingServicePlatform: local
+trainingServicePlatform: local
\ No newline at end of file

test/config_test/multi_phase/multi_phase_tpe.test.yml → test/config/assessors/medianstop.yml

@@ -2,21 +2,24 @@ authorName: nni
 experimentName: default_test
 maxExecDuration: 5m
 maxTrialNum: 8
-trialConcurrency: 4
-searchSpacePath: ./search_space.json
+trialConcurrency: 8
+searchSpacePath: ../naive_trial/search_space.json
 
 tuner:
   builtinTunerName: TPE
   classArgs:
     optimize_mode: maximize
 
+assessor:
+  builtinAssessorName: Medianstop
+  classArgs:
+    optimize_mode: maximize
+
 trial:
-  codeDir: .
-  command: python3 multi_phase.py
+  codeDir: ../naive_trial
+  command: python3 trial.py
   gpuNum: 0
 
 useAnnotation: false
-multiPhase: true
-multiThread: false
 
 trainingServicePlatform: local

test/config_test/examples/mnist-annotation.test.yml → test/config/examples/mnist-annotation.yml

@@ -12,7 +12,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-annotation
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: true

test/config_test/examples/mnist-nested-search-space.test.yml → test/config/examples/mnist-nested-search-space.yml

@@ -14,7 +14,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-nested-search-space
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: false

test/config_test/examples/mnist-pytorch.test.yml → test/config/examples/mnist-pytorch.yml

@@ -13,7 +13,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-pytorch
-  command: python3 mnist.py  --epochs 2
+  command: python3 mnist.py  --epochs 1
   gpuNum: 0
 
 useAnnotation: false

test/config_test/examples/mnist-tfv1.test.yml → test/config/examples/mnist-tfv1.yml

@@ -13,7 +13,7 @@ assessor:
     optimize_mode: maximize
 trial:
   codeDir: ../../../examples/trials/mnist-tfv1
-  command: python3 mnist.py --batch_num 100
+  command: python3 mnist.py --batch_num 10
   gpuNum: 0
 
 useAnnotation: false

test/config/integration_tests.yml

+
+defaultTestCaseConfig:
+  launchCommand: nnictl create --config $configFile
+  stopCommand: nnictl stop
+  experimentStatusCheck: True
+  platform: linux darwin win32
+
+testCases:
+#######################################################################
+# nni examples test
+#######################################################################
+- name: sklearn-classification
+  # test case config yml file relative to nni source code directory
+  configFile: test/config/examples/sklearn-classification.yml
+
+  # test case specific config, the content of configFile will be overrided
+  # by config section
+  config:
+
+  # validator is called after experiment is done
+  # validator class needs to be implemented in nni_test/nnitest/validators.py
+  validator:
+
+  # launch command, default launch command is 'nnictl create --config $configFile'
+  launchCommand: nnictl create --config $configFile
+
+  # stop command, default stop command is 'nnictl stop', empty means no stop command
+  stopCommand: nnictl stop
+
+  # set experiment ID into variable, variable name should start with $, such as $expId
+  setExperimentIdtoVar: $expId
+
+  # check status of experiment before calling validator
+  experimentStatusCheck: True
+
+- name: sklearn-regression
+  configFile: test/config/examples/sklearn-regression.yml
+
+- name: mnist-tfv1
+  configFile: test/config/examples/mnist-tfv1.yml
+
+- name: mnist-keras
+  configFile: test/config/examples/mnist-keras.yml
+
+- name: mnist-pytorch
+  configFile: test/config/examples/mnist-pytorch.yml
+
+- name: mnist-annotation
+  configFile: test/config/examples/mnist-annotation.yml
+
+- name: cifar10-pytorch
+  configFile: test/config/examples/cifar10-pytorch.yml
+  config:
+    # this example downloads large pretrained model weights
+    # test 1 trial to save time
+    maxExecDuration: 10m
+    maxTrialNum: 1
+    trialConcurrency: 1
+    trial:
+      command: python3 main.py --epochs 1 --batches 1
+      gpuNum: 0
+
+- name: nested-ss
+  configFile: test/config/examples/mnist-nested-search-space.yml
+
+
+#########################################################################
+# nni features test
+#########################################################################
+- name: metrics-float
+  configFile: test/config/metrics_test/config.yml
+  config:
+    maxTrialNum: 1
+    trialConcurrency: 1
+  validator:
+    class: MetricsValidator
+    kwargs:
+      expected_result_file: expected_metrics.json
+
+# to be enabled
+#- name: metrics-dict
+#  configFile: test/config/metrics_test/config_dict_metrics.yml
+#  config:
+#    maxTrialNum: 1
+#    trialConcurrency: 1
+#  validator:
+#    class: MetricsValidator
+#    kwargs:
+#      expected_result_file: expected_metrics_dict.json
+
+- name: nnicli
+  configFile: test/config/examples/sklearn-regression.yml
+  config:
+    maxTrialNum: 4
+    trialConcurrency: 4
+  launchCommand: python3 -c 'import nnicli as nc; nc.start_nni("$configFile")'
+  stopCommand: python3 -c 'import nnicli as nc; nc.stop_nni()'
+  validator:
+    class: NnicliValidator
+  platform: linux darwin
+
+- name: foreground
+  configFile: test/config/examples/sklearn-regression.yml
+  launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45
+  stopCommand:
+  experimentStatusCheck: False
+  platform: linux darwin
+
+# Experiment resume test part 1
+- name: nnictl-resume-1
+  configFile: test/config/examples/sklearn-regression.yml
+  setExperimentIdtoVar: $resumeExpId
+
+# Experiment resume test part 2
+- name: nnictl-resume-2
+  configFile: test/config/examples/sklearn-regression.yml
+  launchCommand: nnictl resume $resumeExpId
+
+# Experiment view test
+- name: nnictl-view
+  configFile: test/config/examples/sklearn-regression.yml
+  launchCommand: nnictl view $resumeExpId
+  experimentStatusCheck: False
+
+- name: multi-thread
+  configFile: test/config/multi_thread/config.yml
+
+- name: multi-phase-batch
+  configFile: test/config/multi_phase/batch.yml
+  config:
+    # for batch tuner, maxTrialNum can not exceed length of search space
+    maxTrialNum: 2
+    trialConcurrency: 2
+
+- name: multi-phase-evolution
+  configFile: test/config/multi_phase/evolution.yml
+
+- name: multi-phase-grid
+  configFile: test/config/multi_phase/grid.yml
+  config:
+    maxTrialNum: 2
+    trialConcurrency: 2
+
+- name: multi-phase-metis
+  configFile: test/config/multi_phase/metis.yml
+
+- name: multi-phase-tpe
+  configFile: test/config/multi_phase/tpe.yml
+
+#########################################################################
+# nni assessor test
+#########################################################################
+- name: assessor-curvefitting
+  configFile: test/config/assessors/curvefitting.yml
+
+- name: assessor-medianstop
+  configFile: test/config/assessors/medianstop.yml
+
+#########################################################################
+# nni tuners test
+#########################################################################
+- name: tuner-annel
+  configFile: test/config/tuners/anneal.yml
+
+- name: tuner-evolution
+  configFile: test/config/tuners/evolution.yml
+
+- name: tuner-random
+  configFile: test/config/tuners/random.yml
+
+- name: tuner-smac
+  configFile: test/config/tuners/smac.yml
+  platform: linux darwin
+
+- name: tuner-tpe
+  configFile: test/config/tuners/tpe.yml
+
+- name: tuner-batch
+  configFile: test/config/tuners/batch.yml
+
+- name: tuner-bohb
+  configFile: test/config/tuners/bohb.yml
+  platform: linux darwin
+
+- name: tuner-gp
+  configFile: test/config/tuners/gp.yml
+
+- name: tuner-grid
+  configFile: test/config/tuners/gridsearch.yml
+
+- name: tuner-hyperband
+  configFile: test/config/tuners/hyperband.yml
+
+- name: tuner-metis
+  configFile: test/config/tuners/metis.yml