Merge master into dev-retiarii (#3178)

test/ut/tools/nnictl/mock/experiment.py

@@ -35,7 +35,7 @@ def stop_mock_experiment():
 def generate_args_parser():
     parser = argparse.ArgumentParser()
     parser.add_argument('id', nargs='?')
-    parser.add_argument('--port', '-p', dest='port')
+    parser.add_argument('--port', '-p', type=int, dest='port')
     parser.add_argument('--all', '-a', action='store_true')
     parser.add_argument('--head', type=int)
     parser.add_argument('--tail', type=int)

test/ut/tools/nnictl/test_common_utils.py

@@ -6,6 +6,7 @@ from subprocess import Popen, PIPE, STDOUT
 import sys
 from unittest import TestCase, main, skipIf
 
+sys.path.append(str(Path(__file__).parent))
 from mock.restful_server import init_response
 
 from nni.tools.nnictl.command_utils import kill_command

test/ut/tools/nnictl/test_nnictl_utils.py

 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
+import os
 import sys
+
+sys.path.append(os.path.dirname(__file__))
 from mock.restful_server import init_response
-from mock.experiment import create_mock_experiment, stop_mock_experiment, generate_args_parser, \
-generate_args
+from mock.experiment import create_mock_experiment, stop_mock_experiment, generate_args_parser, generate_args
+
 from nni.tools.nnictl.nnictl_utils import get_experiment_time, get_experiment_status, \
 check_experiment_id, parse_ids, get_config_filename, get_experiment_port, check_rest, \
 trial_ls, list_experiment

ts/nni_manager/training_service/kubernetes/adl/adlTrainingService.ts

@@ -214,10 +214,10 @@ class AdlTrainingService extends KubernetesTrainingService implements Kubernetes
             trialJobId, form, codeDir, outputDir)
         const cleanupScriptTemplate: string =
 `#!/bin/bash
-ps aux | grep "python3 -m nni_trial_tool.trial_keeper" | awk '{print $2}' | xargs kill -2
+ps aux | grep "python3 -m nni.tools.trial_tool.trial_keeper" | awk '{print $2}' | xargs kill -2
 while true;
 do
-    proc=\`ps aux | grep "python3 -m nni_trial_tool.trial_keeper" | awk '{print $2}' | grep "" -c\`
+    proc=\`ps aux | grep "python3 -m nni.tools.trial_tool.trial_keeper" | awk '{print $2}' | grep "" -c\`
     if (( $proc == 1  )); then
         exit 0
     else
@@ -281,7 +281,7 @@ export NNI_TRIAL_SEQ_ID={4}
 mkdir -p $NNI_OUTPUT_DIR
 {5}
 echo '{6}' > $NNI_CODE_DIR/{7}
-python3 -m nni_trial_tool.trial_keeper --trial_command '{8}' \
+python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{8}' \
 --nnimanager_ip {9} --nnimanager_port {10} \
 --nni_manager_version '{11}' --log_collection '{12}'
 `;

ts/nni_manager/training_service/pai/paiYarn/paiYarnData.ts

@@ -16,7 +16,7 @@ fi`;
 export const PAI_TRIAL_COMMAND_FORMAT: string =
 `export NNI_PLATFORM=paiYarn NNI_SYS_DIR={0} NNI_OUTPUT_DIR={1} NNI_TRIAL_JOB_ID={2} NNI_EXP_ID={3} NNI_TRIAL_SEQ_ID={4} MULTI_PHASE={5} \
 && cd $NNI_SYS_DIR && sh install_nni.sh \
-&& python3 -m nni_trial_tool.trial_keeper --trial_command '{6}' --nnimanager_ip '{7}' --nnimanager_port '{8}' \
+&& python3 -m nni.tools.trial_tool.trial_keeper --trial_command '{6}' --nnimanager_ip '{7}' --nnimanager_port '{8}' \
 --pai_hdfs_output_dir '{9}' --pai_hdfs_host '{10}' --pai_user_name {11} --nni_hdfs_exp_dir '{12}' --webhdfs_path '/webhdfs/api/v1' \
 --nni_manager_version '{13}' --log_collection '{14}'`;
 

ts/nni_manager/yarn.lock

@@ -279,6 +279,11 @@
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.3.tgz#bdfd69d61e464dcc81b25159c270d75a73c1a636"
 
+"@types/lockfile@^1.0.0":
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/@types/lockfile/-/lockfile-1.0.1.tgz#434a3455e89843312f01976e010c60f1bcbd56f7"
+  integrity sha512-65WZedEm4AnOsBDdsapJJG42MhROu3n4aSSiu87JXF/pSdlubxZxp3S1yz3kTfkJ2KBPud4CpjoHVAptOm9Zmw==
+
 "@types/mime@*":
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.0.tgz#5a7306e367c539b9f6543499de8dd519fac37a8b"

ts/webui/package.json

@@ -47,6 +47,7 @@
     "react-monaco-editor": "^0.32.1",
     "react-paginate": "^6.3.2",
     "react-pagination": "^1.0.0",
+    "react-responsive": "^8.1.1",
     "react-router": "^5.2.0",
     "react-router-dom": "^5.2.0",
     "react-table": "^7.0.0-rc.15",

ts/webui/src/App.scss

@@ -18,6 +18,7 @@
 
 .headerCon {
     width: 90%;
+    max-width: 1490px;
     margin: 0 auto;
 }
 
@@ -29,9 +30,10 @@
     width: 87%;
     margin: 0 auto;
     min-width: 1200px;
+    max-width: 1490px;
 
-    /* nav bar: 56 + marginTop: 18 */
-    margin-top: 74px;
+    /* nav bar: 56 + marginTop: 24 */
+    margin-top: 80px;
     margin-bottom: 30px;
 }
 

ts/webui/src/components/NavCon.tsx

@@ -157,7 +157,7 @@ class NavCon extends React.Component<NavProps, NavState> {
                         </div>
                         <CommandBarButton
                             iconProps={{ iconName: 'ShowResults' }}
-                            text='Experiment summary'
+                            text='Summary'
                             onClick={this.showExpcontent}
                         />
                         <CommandBarButton iconProps={infoIconAbout} text='About' menuProps={aboutProps} />

ts/webui/src/components/Overview.tsx

@@ -174,12 +174,10 @@ class Overview extends React.Component<{}, OverviewState> {
                                         />
                                     </div>
                                 </div>
-                                <div className='overviewCommand1'>
-                                    <Command1 />
-                                </div>
-                                <div className='overviewCommand2'>
+                                <Stack className='overviewCommand' horizontal>
                                     <Command2 />
-                                </div>
+                                    <Command1 />
+                                </Stack>
                             </div>
                         </div>
                     );

ts/webui/src/components/modals/ExperimentSummaryPanel.tsx

@@ -39,7 +39,7 @@ class ExperimentSummaryPanel extends React.Component<ExpDrawerProps, ExpDrawerSt
         const interResultList = TRIALS.getMetricsList();
         Object.keys(trialMessagesArr).map(item => {
             // not deal with trial's hyperParameters
-            const trialId = trialMessagesArr[item].jobId;
+            const trialId = trialMessagesArr[item].trialJobId;
             // add intermediate result message
             trialMessagesArr[item].intermediate = [];
             Object.keys(interResultList).map(key => {
@@ -94,7 +94,7 @@ class ExperimentSummaryPanel extends React.Component<ExpDrawerProps, ExpDrawerSt
         return (
             <Panel isOpen={true} hasCloseButton={false} isLightDismiss={true} onLightDismissClick={closeExpDrawer}>
                 <div className='panel'>
-                    <div className='panelName'>Experiment summary</div>
+                    <div className='panelName'>Summary</div>
                     <MonacoEditor
                         width='100%'
                         height={monacoEditorHeight}

ts/webui/src/components/overview/command/Command1.tsx

 import React from 'react';
 import { EXPERIMENT } from '../../../static/datamodel';
+import { rightEidtParam } from '../count/commonStyle';
 import '../../../static/style/overview/command.scss';
 
 export const Command1 = (): any => {
@@ -33,7 +34,7 @@ export const Command1 = (): any => {
     }
 
     return (
-        <div className='basic'>
+        <div className='basic' style={rightEidtParam}>
             <div>
                 <p className='command'>Training platform</p>
                 <div className='nowrap'>{EXPERIMENT.profile.params.trainingServicePlatform}</div>

ts/webui/src/components/overview/command/Command2.tsx

 import React from 'react';
 import { TooltipHost, DirectionalHint } from '@fluentui/react';
 import { EXPERIMENT } from '../../../static/datamodel';
+import { leftProgress } from '../count/commonStyle';
 import { TOOLTIP_BACKGROUND_COLOR } from '../../../static/const';
 import '../../../static/style/overview/command.scss';
 
@@ -21,7 +22,7 @@ export const Command2 = (): any => {
         }
     }
     return (
-        <div className='basic'>
+        <div className='basic' style={leftProgress}>
             <p className='command'>Log directory</p>
             <div className='nowrap'>
                 <TooltipHost

ts/webui/src/components/overview/count/EditExperimentParam.tsx

@@ -168,7 +168,7 @@ export const EditExperimentParam = (): any => {
                             return (
                                 <React.Fragment>
                                     <div className={`${editClassName} editparam`}>
-                                        <span>{value.title}</span>
+                                        <div className='title'>{value.title}</div>
                                         <input
                                             className={`${value.field} editparam-Input`}
                                             ref={DurationInputRef}

ts/webui/src/components/overview/count/ExpDuration.tsx

@@ -6,7 +6,7 @@ import { convertDuration, convertTimeAsUnit } from '../../../static/function';
 import { EditExperimentParam } from './EditExperimentParam';
 import { ExpDurationContext } from './ExpDurationContext';
 import { EditExpeParamContext } from './context';
-import { leftProgress, durationItem2, progressHeight } from './commonStyle';
+import { leftProgress, rightEidtParam, progressHeight } from './commonStyle';
 import '../../../static/style/overview/count.scss';
 
 export const ExpDuration = (): any => (
@@ -46,7 +46,7 @@ export const ExpDuration = (): any => (
                             <span>{`${maxExecDurationStr} ${maxDurationUnit}`}</span>
                         </div>
                     </div>
-                    <div style={durationItem2}>
+                    <div style={rightEidtParam}>
                         <EditExpeParamContext.Provider
                             value={{
                                 editType: CONTROLTYPE[0],

ts/webui/src/components/overview/count/TrialCount.tsx

@@ -5,7 +5,7 @@ import { CONTROLTYPE, TOOLTIP_BACKGROUND_COLOR, MAX_TRIAL_NUMBERS } from '../../
 import { EditExperimentParam } from './EditExperimentParam';
 import { EditExpeParamContext } from './context';
 import { ExpDurationContext } from './ExpDurationContext';
-import { leftProgress, trialCountItem2, progressHeight } from './commonStyle';
+import { leftProgress, rightEidtParam, progressHeight } from './commonStyle';
 
 export const TrialCount = (): any => {
     const count = TRIALS.countStatus();
@@ -22,7 +22,7 @@ export const TrialCount = (): any => {
                 const { updateOverviewPage } = value;
                 return (
                     <React.Fragment>
-                        <Stack horizontal horizontalAlign='space-between' className='ExpDuration'>
+                        <Stack horizontal className='ExpDuration'>
                             <div style={leftProgress}>
                                 <TooltipHost
                                     content={`${bar2.toString()} trials`}
@@ -49,7 +49,36 @@ export const TrialCount = (): any => {
                                     <span>{maxTrialNum}</span>
                                 </div>
                             </div>
-                            <div style={trialCountItem2}>
+                        </Stack>
+                        <Stack horizontal className='marginTop'>
+                            <div style={leftProgress}>
+                                <Stack horizontal className='status-count' gap={60}>
+                                    <div>
+                                        <span>Running</span>
+                                        <p>{count.get('RUNNING')}</p>
+                                    </div>
+                                    <div>
+                                        <span>Succeeded</span>
+                                        <p>{count.get('SUCCEEDED')}</p>
+                                    </div>
+                                    <div>
+                                        <span>Stopped</span>
+                                        <p>{stoppedCount}</p>
+                                    </div>
+                                </Stack>
+                                <Stack horizontal className='status-count marginTop' gap={80}>
+                                    <div>
+                                        <span>Failed</span>
+                                        <p>{count.get('FAILED')}</p>
+                                    </div>
+                                    <div>
+                                        <span>Waiting</span>
+                                        <p>{count.get('WAITING')}</p>
+                                    </div>
+                                </Stack>
+                            </div>
+
+                            <div style={rightEidtParam}>
                                 <EditExpeParamContext.Provider
                                     value={{
                                         title: MAX_TRIAL_NUMBERS,
@@ -65,42 +94,22 @@ export const TrialCount = (): any => {
                                         <EditExperimentParam />
                                     </div>
                                 </EditExpeParamContext.Provider>
-                                <EditExpeParamContext.Provider
-                                    value={{
-                                        title: 'Concurrency',
-                                        field: 'trialConcurrency',
-                                        editType: CONTROLTYPE[2],
-                                        // maxExecDuration: EXPERIMENT.profile.params.maxExecDuration,
-                                        maxExecDuration: '',
-                                        maxTrialNum: EXPERIMENT.profile.params.maxTrialNum,
-                                        trialConcurrency: EXPERIMENT.profile.params.trialConcurrency,
-                                        updateOverviewPage
-                                    }}
-                                >
-                                    <EditExperimentParam />
-                                </EditExpeParamContext.Provider>
-                            </div>
-                        </Stack>
-                        <Stack horizontal horizontalAlign='space-between' className='trialStatus'>
-                            <div className='basic'>
-                                <p>Running</p>
-                                <div>{count.get('RUNNING')}</div>
-                            </div>
-                            <div className='basic'>
-                                <p>Succeeded</p>
-                                <div>{count.get('SUCCEEDED')}</div>
-                            </div>
-                            <div className='basic'>
-                                <p>Stopped</p>
-                                <div>{stoppedCount}</div>
-                            </div>
-                            <div className='basic'>
-                                <p>Failed</p>
-                                <div>{count.get('FAILED')}</div>
-                            </div>
-                            <div className='basic'>
-                                <p>Waiting</p>
-                                <div>{count.get('WAITING')}</div>
+                                <div className='concurrency'>
+                                    <EditExpeParamContext.Provider
+                                        value={{
+                                            title: 'Concurrency',
+                                            field: 'trialConcurrency',
+                                            editType: CONTROLTYPE[2],
+                                            // maxExecDuration: EXPERIMENT.profile.params.maxExecDuration,
+                                            maxExecDuration: '',
+                                            maxTrialNum: EXPERIMENT.profile.params.maxTrialNum,
+                                            trialConcurrency: EXPERIMENT.profile.params.trialConcurrency,
+                                            updateOverviewPage
+                                        }}
+                                    >
+                                        <EditExperimentParam />
+                                    </EditExpeParamContext.Provider>
+                                </div>
                             </div>
                         </Stack>
                     </React.Fragment>

ts/webui/src/components/overview/count/commonStyle.ts

 const leftProgress: React.CSSProperties = {
-    width: '33%',
-    position: 'relative',
-    top: 6
+    width: '60%',
+    position: 'relative'
 };
-const durationItem2: React.CSSProperties = {
-    width: '51.5%',
-    paddingLeft: '15%'
+const rightEidtParam: React.CSSProperties = {
+    paddingLeft: '9%',
+    boxSizing: 'border-box'
 };
 
-const trialCountItem2: React.CSSProperties = {
-    width: '51.5%'
-};
 const progressHeight = 8;
-export { leftProgress, durationItem2, trialCountItem2, progressHeight };
+
+export { leftProgress, rightEidtParam, progressHeight };

ts/webui/src/components/overview/experiment/BasicInfo.tsx

@@ -23,11 +23,11 @@ export const ReBasicInfo = (): any => {
 
     return (
         <div>
-            <Stack horizontal horizontalAlign='space-between' className='mess'>
+            <Stack horizontal horizontalAlign='space-between' className='marginTop'>
                 <div className='basic'>
                     <p>Name</p>
                     <div className='nowrap'>{EXPERIMENT.profile.params.experimentName}</div>
-                    <p className='margin'>ID</p>
+                    <p className='marginTop'>ID</p>
                     <div className='nowrap'>{EXPERIMENT.profile.id}</div>
                 </div>
                 <div className='basic'>
@@ -76,7 +76,7 @@ export const ReBasicInfo = (): any => {
                     <BestMetricContext.Consumer>
                         {(value): React.ReactNode => (
                             <Stack className='bestMetric'>
-                                <p className='margin'>Best metric</p>
+                                <p className='marginTop'>Best metric</p>
                                 <div className={EXPERIMENT.status}>
                                     {isNaN(value.bestAccuracy) ? 'N/A' : value.bestAccuracy.toFixed(6)}
                                 </div>
@@ -87,7 +87,7 @@ export const ReBasicInfo = (): any => {
                 <div className='basic'>
                     <p>Start time</p>
                     <div className='nowrap'>{formatTimestamp(EXPERIMENT.profile.startTime)}</div>
-                    <p className='margin'>End time</p>
+                    <p className='marginTop'>End time</p>
                     <div className='nowrap'>{formatTimestamp(EXPERIMENT.profile.endTime)}</div>
                 </div>
             </Stack>

ts/webui/src/components/overview/table/SuccessTable.tsx

@@ -145,7 +145,7 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState>
             key: 'id',
             fieldName: 'id',
             minWidth: 60,
-            maxWidth: 118,
+            maxWidth: 90,
             isResizable: true,
             className: 'tableHead leftTitle',
             data: 'string',
@@ -155,8 +155,8 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState>
         {
             name: 'Duration',
             key: 'duration',
-            minWidth: 85,
-            maxWidth: 166,
+            minWidth: 80,
+            maxWidth: 120,
             isResizable: true,
             fieldName: 'duration',
             data: 'number',
@@ -170,8 +170,8 @@ class SuccessTable extends React.Component<SuccessTableProps, SuccessTableState>
         {
             name: 'Status',
             key: 'status',
-            minWidth: 98,
-            maxWidth: 160,
+            minWidth: 88,
+            maxWidth: 120,
             isResizable: true,
             fieldName: 'status',
             onRender: (item: any): React.ReactNode => (

ts/webui/src/components/slideNav/SlideNavBtns.tsx

 import React, { useState, useCallback } from 'react';
-import { Stack } from '@fluentui/react';
+import { Stack, DefaultButton, Icon } from '@fluentui/react';
+import MediaQuery from 'react-responsive';
 import TrialConfigPanel from './TrialConfigPanel';
 import LogPanel from '../modals/LogPanel';
 import IconButtonTemplate from './IconButtonTemplet';
@@ -28,9 +29,31 @@ export const SlideNavBtns = (): any => {
         // right side nav buttons
         <React.Fragment>
             <Stack className='config'>
-                <IconButtonTemplate icon='DocumentSearch' btuName='Search space' event={showSearchSpacePanel} />
-                <IconButtonTemplate icon='Archive' btuName='Config' event={showTrialConfigpPanel} />
-                <IconButtonTemplate icon='FilePDB' btuName='Log files' event={showLogPanel} />
+                <MediaQuery maxWidth={1799}>
+                    <IconButtonTemplate icon='DocumentSearch' btuName='Search space' event={showSearchSpacePanel} />
+                    <IconButtonTemplate icon='Archive' btuName='Config' event={showTrialConfigpPanel} />
+                    <IconButtonTemplate icon='FilePDB' btuName='Log files' event={showLogPanel} />
+                </MediaQuery>
+                <MediaQuery minWidth={1798}>
+                    <div className='container'>
+                        <DefaultButton onClick={showSearchSpacePanel} className='maxScrBtn'>
+                            <Icon iconName='DocumentSearch' />
+                            <span className='margin'>Search space</span>
+                        </DefaultButton>
+                    </div>
+                    <div className='container'>
+                        <DefaultButton onClick={showTrialConfigpPanel} className='maxScrBtn configBtn'>
+                            <Icon iconName='Archive' />
+                            <span className='margin'>Config</span>
+                        </DefaultButton>
+                    </div>
+                    <div className='container'>
+                        <DefaultButton onClick={showLogPanel} className='maxScrBtn logBtn'>
+                            <Icon iconName='FilePDB' />
+                            <span className='margin'>Log files</span>
+                        </DefaultButton>
+                    </div>
+                </MediaQuery>
             </Stack>
             {isShowConfigPanel && <TrialConfigPanel panelName={panelName} hideConfigPanel={hideConfigPanel} />}
             {/* the panel for dispatcher & nnimanager log message */}