Merge pull request #150 from Microsoft/master

merge master

examples/trials/mnist-keras/config_pai.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution, BatchTuner
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
   #SMAC (SMAC should be installed through nnictl)
   builtinTunerName: TPE
   classArgs:

examples/trials/mnist/config.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution, BatchTuner
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
   #SMAC (SMAC should be installed through nnictl)
   builtinTunerName: TPE
   classArgs:

examples/trials/mnist/config_assessor.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution, BatchTuner
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
   #SMAC (SMAC should be installed through nnictl)
   builtinTunerName: TPE
   classArgs:

examples/trials/mnist/config_frameworkcontroller.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
   builtinTunerName: TPE
   classArgs:
     #choice: maximize, minimize

examples/trials/mnist/config_kubeflow.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
   builtinTunerName: TPE
   classArgs:
     #choice: maximize, minimize

examples/trials/mnist/config_pai.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution, BatchTuner
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
   #SMAC (SMAC should be installed through nnictl)
   builtinTunerName: TPE
   classArgs:

examples/trials/network_morphism/README.md

@@ -5,7 +5,7 @@ The Network Morphism is a build-in Tuner using network morphism techniques to se
 
 ### 1. Training framework support
 
-The network morphism now is framework-based, and we have not implemented the framework-free methods. The training frameworks which we have supported yet are Pytorch and Keras. If you get familiar with the intermediate JSON format, you can build your own model in your own training framework. In the future, we will change to intermediate format from JSON to ONNX in order to get a [standard intermediate representation spec](https://github.com/onnx/onnx/blob/master/docs/IR.md).
+The network morphism now is framework-based, and we have not implemented the framework-free methods. The training frameworks which we have supported yet are PyTorch and Keras. If you get familiar with the intermediate JSON format, you can build your own model in your own training framework. In the future, we will change to intermediate format from JSON to ONNX in order to get a [standard intermediate representation spec](https://github.com/onnx/onnx/blob/master/docs/IR.md).
 
 
 ### 2. Install the requirements 

examples/trials/sklearn/classification/config.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
   builtinTunerName: TPE
   classArgs:
     #choice: maximize, minimize

examples/trials/sklearn/classification/config_pai.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution, BatchTuner
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner,MetisTuner
   #SMAC (SMAC should be installed through nnictl)
   builtinTunerName: TPE
   classArgs:

examples/trials/sklearn/regression/config.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
   builtinTunerName: TPE
   classArgs:
     #choice: maximize, minimize

examples/trials/sklearn/regression/config_pai.yml

@@ -9,7 +9,7 @@ searchSpacePath: search_space.json
 #choice: true, false
 useAnnotation: false
 tuner:
-  #choice: TPE, Random, Anneal, Evolution, BatchTuner
+  #choice: TPE, Random, Anneal, Evolution, BatchTuner, MetisTuner
   #SMAC (SMAC should be installed through nnictl)
   builtinTunerName: TPE
   classArgs:

src/nni_manager/training_service/local/localTrainingService.ts

@@ -103,7 +103,7 @@ class LocalTrainingService implements TrainingService {
     protected log: Logger;
     protected localTrailConfig?: TrialConfig;
     private isMultiPhase: boolean = false;
-    private streams: Array<ts.Stream>;
+    protected jobStreamMap: Map<string, ts.Stream>;
 
     constructor() {
         this.eventEmitter = new EventEmitter();
@@ -113,7 +113,7 @@ class LocalTrainingService implements TrainingService {
         this.stopping = false;
         this.log = getLogger();
         this.trialSequenceId = -1;
-        this.streams = new Array<ts.Stream>();
+        this.jobStreamMap = new Map<string, ts.Stream>();
         this.log.info('Construct local machine training service.');
     }
 
@@ -307,14 +307,24 @@ class LocalTrainingService implements TrainingService {
     public cleanUp(): Promise<void> {
         this.log.info('Stopping local machine training service...');
         this.stopping = true;
-        for (const stream of this.streams) {
+        for (const stream of this.jobStreamMap.values()) {
             stream.destroy();
         }
         return Promise.resolve();
     }
 
     protected onTrialJobStatusChanged(trialJob: TrialJobDetail, oldStatus: TrialJobStatus): void {
-        //abstract
+        //if job is not running, destory job stream
+        if(['SUCCEEDED', 'FAILED', 'USER_CANCELED', 'SYS_CANCELED', 'EARLY_STOPPED'].includes(trialJob.status)) {
+            if(this.jobStreamMap.has(trialJob.id)) {
+                const stream = this.jobStreamMap.get(trialJob.id);
+                if(!stream) {
+                    throw new Error(`Could not find stream in trial ${trialJob.id}`);
+                }
+                stream.destroy();
+                this.jobStreamMap.delete(trialJob.id);
+            }
+        }
     }
 
     protected getEnvironmentVariables(trialJobDetail: TrialJobDetail, _: {}): { key: string; value: string }[] {
@@ -396,7 +406,8 @@ class LocalTrainingService implements TrainingService {
                 buffer = remain;
             }
         });
-        this.streams.push(stream);
+        
+        this.jobStreamMap.set(trialJobDetail.id, stream);
     }
 
     private async runHostJob(form: HostJobApplicationForm): Promise<TrialJobDetail> {

src/nni_manager/training_service/local/localTrainingServiceForGPU.ts

@@ -78,6 +78,7 @@ class LocalTrainingServiceForGPU extends LocalTrainingService {
     }
 
     protected onTrialJobStatusChanged(trialJob: LocalTrialJobDetailForGPU, oldStatus: TrialJobStatus): void {
+        super.onTrialJobStatusChanged(trialJob, oldStatus);
         if (trialJob.gpuIndices !== undefined && trialJob.gpuIndices.length !== 0 && this.gpuScheduler !== undefined) {
             if (oldStatus === 'RUNNING' && trialJob.status !== 'RUNNING') {
                 for (const index of trialJob.gpuIndices) {

src/webui/src/components/Overview.tsx

@@ -33,6 +33,8 @@ interface OverviewState {
     isTop10: boolean;
     titleMaxbgcolor: string;
     titleMinbgcolor?: string;
+    // trial stdout is content(false) or link(true)
+    isLogCollection: boolean;
 }
 
 class Overview extends React.Component<{}, OverviewState> {
@@ -76,7 +78,8 @@ class Overview extends React.Component<{}, OverviewState> {
                 totalCurrentTrial: 0
             },
             isTop10: true,
-            titleMaxbgcolor: '#999'
+            titleMaxbgcolor: '#999',
+            isLogCollection: false
         };
     }
 
@@ -95,6 +98,12 @@ class Overview extends React.Component<{}, OverviewState> {
                     const endTimenum = sessionData.endTime;
                     const assessor = sessionData.params.assessor;
                     const advisor = sessionData.params.advisor;
+                     // default logCollection is true
+                    const logCollection = res.data.params.logCollection;
+                    let expLogCollection: boolean = false;
+                    if (logCollection !== undefined && logCollection !== 'none') {
+                        expLogCollection = true;
+                    }
                     trialPro.push({
                         id: sessionData.id,
                         author: sessionData.params.authorName,
@@ -111,7 +120,8 @@ class Overview extends React.Component<{}, OverviewState> {
                         tuner: sessionData.params.tuner,
                         assessor: assessor ? assessor : undefined,
                         advisor: advisor ? advisor : undefined,
-                        clusterMetaData: clusterMetaData ? clusterMetaData : undefined
+                        clusterMetaData: clusterMetaData ? clusterMetaData : undefined,
+                        logCollection: logCollection
                     });
                     // search space format loguniform max and min
                     const temp = sessionData.params.searchSpace;
@@ -134,7 +144,8 @@ class Overview extends React.Component<{}, OverviewState> {
                     if (this._isMounted) {
                         this.setState({
                             trialProfile: trialPro[0],
-                            searchSpace: searchSpace
+                            searchSpace: searchSpace,
+                            isLogCollection: expLogCollection
                         });
                     }
                 }
@@ -379,7 +390,7 @@ class Overview extends React.Component<{}, OverviewState> {
         const {
             trialProfile, searchSpace, tableData, accuracyData,
             accNodata, status, errorStr, trialNumber, bestAccuracy,
-            titleMaxbgcolor, titleMinbgcolor
+            titleMaxbgcolor, titleMinbgcolor, isLogCollection
         } = this.state;
 
         return (
@@ -410,7 +421,7 @@ class Overview extends React.Component<{}, OverviewState> {
                         </Row>
                     </Col>
                     <Col span={8} className="overviewBoder">
-                        <Title1 text="Trial Profile" icon="4.png" />
+                        <Title1 text="Profile" icon="4.png" />
                         <Row className="experiment">
                             {/* the scroll bar all the trial profile in the searchSpace div*/}
                             <div className="experiment searchSpace">
@@ -453,6 +464,7 @@ class Overview extends React.Component<{}, OverviewState> {
                     <Col span={16} id="succeTable">
                         <SuccessTable
                             tableSource={tableData}
+                            logCollection={isLogCollection}
                             trainingPlatform={trialProfile.trainingServicePlatform}
                         />
                     </Col>

src/webui/src/components/TrialsDetail.tsx

@@ -21,8 +21,9 @@ interface TrialDetailState {
     searchResultSource: Array<TableObj>;
     isHasSearch: boolean;
     experimentStatus: string;
-    entriesTable: number;
     experimentPlatform: string;
+    experimentLogCollection: boolean;
+    entriesTable: number;
     searchSpace: string;
     defaultMetric?: Array<number>;
 }
@@ -49,7 +50,6 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
     );
 
     private titleOfIntermediate = (
-        // <Title1 text="Intermediate Result" icon="intermediate.png" />
         <div className="panelTitle">
             <Icon type="line-chart" />
             <span>Intermediate Result</span>
@@ -65,9 +65,10 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
             tableListSource: [],
             searchResultSource: [],
             experimentStatus: '',
+            experimentPlatform: '',
+            experimentLogCollection: false,
             entriesTable: 20,
             isHasSearch: false,
-            experimentPlatform: '',
             searchSpace: '',
             defaultMetric: [0, 1]
         };
@@ -279,6 +280,7 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
         alert('TableList component was not properly initialized.');
     }
 
+    // get and set logCollection val
     checkExperimentPlatform = () => {
         axios(`${MANAGER_IP}/experiment`, {
             method: 'GET'
@@ -290,10 +292,17 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
                         res.data.params.trainingServicePlatform
                         :
                         '';
+                    // default logCollection is true
+                    const logCollection = res.data.params.logCollection;
+                    let expLogCollection: boolean = false;
+                    if (logCollection !== undefined && logCollection !== 'none') {
+                        expLogCollection = true;
+                    }
                     if (this._isMounted) {
                         this.setState({
                             experimentPlatform: trainingPlatform,
-                            searchSpace: res.data.params.searchSpace
+                            searchSpace: res.data.params.searchSpace,
+                            experimentLogCollection: expLogCollection
                         });
                     }
                 }
@@ -318,7 +327,7 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
         const {
             tableListSource, searchResultSource, isHasSearch,
             entriesTable, experimentPlatform, searchSpace,
-            defaultMetric
+            defaultMetric, experimentLogCollection
         } = this.state;
         const source = isHasSearch ? searchResultSource : tableListSource;
         return (
@@ -392,8 +401,9 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
                 <TableList
                     entries={entriesTable}
                     tableSource={source}
-                    updateList={this.getDetailSource}
                     platform={experimentPlatform}
+                    updateList={this.getDetailSource}
+                    logCollection={experimentLogCollection}
                     ref={(tabList) => this.tableList = tabList}
                 />
             </div>

src/webui/src/components/overview/SuccessTable.tsx

 import * as React from 'react';
-import axios from 'axios';
-import { Modal, Input, Table } from 'antd';
-const { TextArea } = Input;
+import { Table } from 'antd';
 import OpenRow from '../public-child/OpenRow';
 import DefaultMetric from '../public-child/DefaultMetrc';
-import { DOWNLOAD_IP } from '../../static/const';
 import { TableObj } from '../../static/interface';
 import { convertDuration } from '../../static/function';
 import '../../static/style/tableStatus.css';
@@ -13,72 +10,25 @@ import '../../static/style/tableList.scss';
 interface SuccessTableProps {
     tableSource: Array<TableObj>;
     trainingPlatform: string;
+    logCollection: boolean;
 }
 
-interface SuccessTableState {
-    isShowLogModal: boolean;
-    logContent: string;
-}
-
-class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState> {
+class SuccessTable extends React.Component<SuccessTableProps, {}> {
 
     public _isMounted = false;
 
     constructor(props: SuccessTableProps) {
         super(props);
 
-        this.state = {
-            isShowLogModal: false,
-            logContent: ''
-        };
-
-    }
-
-    showLogModalOverview = (id: string) => {
-        axios(`${DOWNLOAD_IP}/trial_${id}.log`, {
-            method: 'GET'
-        })
-            .then(res => {
-                if (res.status === 200) {
-                    if (this._isMounted) {
-                        this.setState(() => ({
-                            logContent: res.data
-                        }));
-                    }
-                }
-            })
-            .catch(error => {
-                if (error.response.status === 500) {
-                    if (this._isMounted) {
-                        this.setState(() => ({
-                            logContent: 'failed to get log message'
-                        }));
-                    }
-                }
-            });
-        if (this._isMounted) {
-            this.setState({
-                isShowLogModal: true
-            });
-        }
-    }
-
-    hideLogModalOverview = () => {
-        if (this._isMounted) {
-            this.setState({
-                isShowLogModal: false,
-                logContent: '' // close modal, delete data
-            });
-        }
     }
 
     openRow = (record: TableObj) => {
-        const { trainingPlatform } = this.props;
+        const { trainingPlatform, logCollection } = this.props;
         return (
             <OpenRow
-                showLogModalOverview={this.showLogModalOverview}
                 trainingPlatform={trainingPlatform}
                 record={record}
+                logCollection={logCollection}
             />
         );
     }
@@ -93,7 +43,6 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState>
 
     render() {
         const { tableSource } = this.props;
-        const { isShowLogModal, logContent } = this.state;
 
         let bgColor = '';
         const columns = [{
@@ -168,25 +117,9 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState>
                     className="commonTableStyle"
                     pagination={false}
                 />
-                {/* trial log modal */}
-                <Modal
-                    title="trial log"
-                    visible={isShowLogModal}
-                    onCancel={this.hideLogModalOverview}
-                    footer={null}
-                    destroyOnClose={true}
-                    width="80%"
-                >
-                    <div id="trialLogContent" style={{ height: window.innerHeight * 0.6 }}>
-                        <TextArea
-                            value={logContent}
-                            disabled={true}
-                            className="logcontent"
-                        />
-                    </div>
-                </Modal>
             </div >
         );
     }
 }
+
 export default SuccessTable;

src/webui/src/components/overview/TrialProfile.tsx

@@ -22,6 +22,7 @@ class TrialInfo extends React.Component<TrialInfoProps, {}> {
             trialConcurrency: tiralProInfo.runConcurren,
             tuner: tiralProInfo.tuner,
             assessor: tiralProInfo.assessor ? tiralProInfo.assessor : undefined,
+            logCollection: tiralProInfo.logCollection ? tiralProInfo.logCollection : undefined,
             advisor: tiralProInfo.advisor ? tiralProInfo.advisor : undefined,
             clusterMetaData: tiralProInfo.clusterMetaData ? tiralProInfo.clusterMetaData : undefined
         });

src/webui/src/components/public-child/DefaultMetrc.tsx

@@ -19,7 +19,7 @@ class DefaultMetric extends React.Component<DefaultMetricProps, {}> {
             accuracy = record.acc.default;
         }
         let wei = 0;
-        if (accuracy) {
+        if (accuracy !== undefined) {
             if (accuracy.toString().indexOf('.') !== -1) {
                 wei = accuracy.toString().length - accuracy.toString().indexOf('.') - 1;
             }
@@ -27,7 +27,7 @@ class DefaultMetric extends React.Component<DefaultMetricProps, {}> {
         return (
             <div>
                 {
-                    record.acc && record.acc.default
+                    record.acc !== undefined && record.acc.default !== undefined
                         ?
                         wei > 6
                             ?

src/webui/src/components/public-child/IntermediateVal.tsx

+import * as React from 'react';
+import { TableObj } from '../../static/interface';
+
+interface IntermediateValProps {
+    record: TableObj;
+}
+
+class IntermediateVal extends React.Component<IntermediateValProps, {}> {
+
+    constructor(props: IntermediateValProps) {
+        super(props);
+
+    }
+
+    render() {
+        const { record } = this.props;
+        const interArr = record.description.intermediate;
+        const status = record.status;
+        let lastVal;
+        let wei = 0;
+        if (interArr !== undefined) {
+            lastVal = interArr[interArr.length - 1];
+        }
+        let result: string = JSON.stringify(lastVal);
+        if (lastVal !== undefined) {
+            if (lastVal.toString().indexOf('.') !== -1) {
+                wei = lastVal.toString().length - lastVal.toString().indexOf('.') - 1;
+                if (wei > 6) {
+                    result = `${lastVal.toFixed(6)}`;
+                }
+                if (status === 'SUCCEEDED') {
+                    result = `${lastVal.toFixed(6)} (FINAL)`;
+                } else {
+                    result = `${lastVal.toFixed(6)} (LATEST)`;
+                }
+
+            }
+        } else {
+            result = '--';
+        }
+        return (
+            <div>{result}</div>
+        );
+    }
+}
+
+export default IntermediateVal;

src/webui/src/components/public-child/LogPathChild.tsx

@@ -17,19 +17,15 @@ class LogPathChild extends React.Component<LogpathChildProps, {}> {
         const isLink = /^http/gi.test(eachLogpath);
 
         return (
-            <div>
+            <div className="logpath">
+                <span className="logName">{logName}</span>
                 {
                     isLink
                         ?
-                        <div className="logpath">
-                            <span className="logName">{logName}</span>
                         <a className="logContent logHref" href={eachLogpath} target="_blank">{eachLogpath}</a>
-                        </div>
                         :
-                        <div className="logpath">
-                            <span className="logName">{logName}</span>
                         <span className="logContent">{eachLogpath}</span>
-                        </div>
+
                 }
             </div>
         );