Show trial error message (#2424)

src/webui/src/App.tsx

@@ -21,7 +21,7 @@ class App extends React.Component<{}, AppState> {
     private timerId!: number | undefined;
     private dataFormatimer!: number;
     private firstLoad: boolean = false; // when click refresh selector options
-
+    
     constructor(props: {}) {
         super(props);
         this.state = {
@@ -49,8 +49,8 @@ class App extends React.Component<{}, AppState> {
     }
 
     getFinalDataFormat = (): void => {
-        for(let i = 0; this.state.isillegalFinal === false; i++){
-            if(TRIALS.succeededTrials()[0] !== undefined && TRIALS.succeededTrials()[0].final !== undefined){
+        for (let i = 0; this.state.isillegalFinal === false; i++) {
+            if (TRIALS.succeededTrials()[0] !== undefined && TRIALS.succeededTrials()[0].final !== undefined) {
                 // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                 const oneSucceedTrial = JSON.parse(JSON.parse(TRIALS.succeededTrials()[0].final!.data));
                 if (typeof oneSucceedTrial === 'number' || oneSucceedTrial.hasOwnProperty('default')) {
@@ -71,14 +71,14 @@ class App extends React.Component<{}, AppState> {
     }
 
     changeInterval = (interval: number): void => {
-        
+
         window.clearTimeout(this.timerId);
         if (interval === 0) {
-            return;   
+            return;
         }
         // setState will trigger page refresh at once.
         // setState is asyc, interval not update to (this.state.interval) at once.
-        this.setState({interval}, () => {
+        this.setState({ interval }, () => {
             this.firstLoad = true;
             this.refresh();
         });
@@ -96,7 +96,7 @@ class App extends React.Component<{}, AppState> {
 
     // overview best trial module
     changeEntries = (entries: string): void => {
-        this.setState({bestTrialEntries: entries});
+        this.setState({ bestTrialEntries: entries });
     }
 
     render(): React.ReactNode {
@@ -106,15 +106,25 @@ class App extends React.Component<{}, AppState> {
         if (experimentUpdateBroadcast === 0 || trialsUpdateBroadcast === 0) {
             return null;  // TODO: render a loading page
         }
+        
+        const errorList = [
+            { errorWhere: TRIALS.jobListError(), errorMessage: TRIALS.getJobErrorMessage() },
+            { errorWhere: EXPERIMENT.experimentError(), errorMessage: EXPERIMENT.getExperimentMessage() },
+            { errorWhere: EXPERIMENT.statusError(), errorMessage: EXPERIMENT.getStatusMessage() },
+            { errorWhere: TRIALS.MetricDataError(), errorMessage: TRIALS.getMetricDataErrorMessage() },
+            { errorWhere: TRIALS.latestMetricDataError(), errorMessage: TRIALS.getLatestMetricDataErrorMessage() },
+            { errorWhere: TRIALS.metricDataRangeError(), errorMessage: TRIALS.metricDataRangeErrorMessage() }
+        ];
+
         const reactPropsChildren = React.Children.map(this.props.children, child =>
             React.cloneElement(
                 child as React.ReactElement<any>, {
-                    interval,
-                    columnList, changeColumn: this.changeColumn,
-                    experimentUpdateBroadcast,
-                    trialsUpdateBroadcast,
-                    metricGraphMode, changeMetricGraphMode: this.changeMetricGraphMode,
-                    bestTrialEntries, changeEntries: this.changeEntries
+                interval,
+                columnList, changeColumn: this.changeColumn,
+                experimentUpdateBroadcast,
+                trialsUpdateBroadcast,
+                metricGraphMode, changeMetricGraphMode: this.changeMetricGraphMode,
+                bestTrialEntries, changeEntries: this.changeEntries
             })
         );
 
@@ -127,6 +137,16 @@ class App extends React.Component<{}, AppState> {
                 </div>
                 <Stack className="contentBox">
                     <Stack className="content">
+                        {/* if api has error field, show error message */}
+                        {
+                            errorList.map((item, key) => {
+                                return (
+                                    item.errorWhere && <div key={key} className="warning">
+                                        <MessageInfo info={item.errorMessage} typeInfo="error" />
+                                    </div>
+                                );
+                            })
+                        }
                         {isillegalFinal && <div className="warning">
                             <MessageInfo info={expWarningMessage} typeInfo="warning" />
                         </div>}
@@ -149,18 +169,20 @@ class App extends React.Component<{}, AppState> {
             if (trialsUpdated) {
                 this.setState(state => ({ trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 }));
             }
+
         } else {
             this.firstLoad = false;
         }
 
-        if (['DONE', 'ERROR', 'STOPPED'].includes(EXPERIMENT.status)) {
+        // experiment status and /trial-jobs api's status could decide website update
+        if (['DONE', 'ERROR', 'STOPPED'].includes(EXPERIMENT.status) || TRIALS.jobListError()) {
             // experiment finished, refresh once more to ensure consistency
             this.setState({ interval: 0 });
             this.lastRefresh();
             return;
         }
 
-        this.timerId =  window.setTimeout(this.refresh, this.state.interval * 1000);
+        this.timerId = window.setTimeout(this.refresh, this.state.interval * 1000);
 
     }
 

src/webui/src/components/Modals/MessageInfo.tsx

@@ -18,7 +18,7 @@ class MessageInfo extends React.Component<MessageInfoProps, {}> {
         return (
             <MessageBar
                 messageBarType={MessageBarType[typeInfo]}
-                isMultiline={false}
+                isMultiline={true}
                 className={className}
             >
                 {info}

src/webui/src/components/overview/ProgressItem.tsx

@@ -22,7 +22,7 @@ class ProgressBar extends React.Component<ProItemProps, {}> {
             <div>
                 <Stack horizontal className={`probar ${bgclass}`}>
                     <div className="name">{who}</div>
-                    <div className="showProgress" style={{ width: '80%' }}>
+                    <div className="showProgress" style={{ width: '78%' }}>
                         <ProgressIndicator
                             barHeight={30}
                             percentComplete={percent}
@@ -32,7 +32,7 @@ class ProgressBar extends React.Component<ProItemProps, {}> {
                             <StackItem className="right" grow={70}>{maxString}</StackItem>
                         </Stack>
                     </div>
-                    <div className="description" style={{ width: '20%' }}>{description}</div>
+                    <div className="description" style={{ width: '22%' }}>{description}</div>
                 </Stack>
                 <br />
             </div>

src/webui/src/static/function.ts

@@ -3,6 +3,19 @@ import axios from 'axios';
 import { MANAGER_IP } from './const';
 import { MetricDataRecord, FinalType, TableObj } from './interface';
 
+async function requestAxios(url: string) {
+    const response = await axios.get(url);
+    if (response.status === 200) {
+        if (response.data.error !== undefined) {
+            throw new Error(`API ${url} ${response.data.error}`);
+        } else {
+            return response.data as any;
+        }
+    } else {
+        throw new Error(`API ${url} ${response.status} error`);
+    }
+}
+
 const convertTime = (num: number): string => {
     if (num <= 0) {
         return '0';
@@ -219,5 +232,5 @@ export {
     convertTime, convertDuration, getFinalResult, getFinal, downFile,
     intermediateGraphOption, killJob, filterByStatus, filterDuration,
     formatAccuracy, formatTimestamp, metricAccuracy, parseMetrics,
-    isArrayType
+    isArrayType, requestAxios
 };

src/webui/src/static/model/experiment.ts

 import axios from 'axios';
 import { MANAGER_IP } from '../const';
 import { ExperimentProfile, NNIManagerStatus } from '../interface';
+import { requestAxios } from '../function';
 
 function compareProfiles(profile1?: ExperimentProfile, profile2?: ExperimentProfile): boolean {
     if (!profile1 || !profile2) {
@@ -14,32 +15,87 @@ function compareProfiles(profile1?: ExperimentProfile, profile2?: ExperimentProf
 class Experiment {
     private profileField?: ExperimentProfile = undefined;
     private statusField?: NNIManagerStatus = undefined;
+    private isexperimentError: boolean = false;
+    private experimentErrorMessage: string = '';
+    private isStatusError: boolean = false;
+    private statusErrorMessage: string = '';
 
     public async init(): Promise<void> {
         while (!this.profileField || !this.statusField) {
+            if (this.isexperimentError) {
+                return;
+            }
+            if (this.isStatusError) {
+                return;
+            }
             await this.update();
         }
     }
 
+    public experimentError(): boolean {
+        return this.isexperimentError;
+    }
+
+    public statusError(): boolean {
+        return this.isStatusError;
+    }
+
+    public getExperimentMessage(): string {
+        return this.experimentErrorMessage;
+    }
+
+    public getStatusMessage(): string {
+        return this.statusErrorMessage;
+    }
+
     public async update(): Promise<boolean> {
-        const profilePromise = axios.get(`${MANAGER_IP}/experiment`);
-        const statusPromise = axios.get(`${MANAGER_IP}/check-status`);
-        const [ profileResponse, statusResponse ] = await Promise.all([ profilePromise, statusPromise ]);
         let updated = false;
-        if (statusResponse.status === 200) {
-            updated = JSON.stringify(this.statusField) === JSON.stringify(statusResponse.data);
-            this.statusField = statusResponse.data;
-        }
-        if (profileResponse.status === 200) {
-            updated = updated || compareProfiles(this.profileField, profileResponse.data);
-            this.profileField = profileResponse.data;
-        }
+
+        await requestAxios(`${MANAGER_IP}/experiment`)
+            .then(data => {
+                updated = updated || compareProfiles(this.profileField, data);
+                this.profileField = data;
+            })
+            .catch(error => {
+                this.isexperimentError = true;
+                this.experimentErrorMessage = `${error.message}`;
+                updated = true;
+            });
+
+        await requestAxios(`${MANAGER_IP}/check-status`)
+            .then(data => {
+                updated = JSON.stringify(this.statusField) === JSON.stringify(data);
+                this.statusField = data;
+            })
+            .catch(error => {
+                this.isStatusError = true;
+                this.statusErrorMessage = `${error.message}`;
+                updated = true;
+            });
+
         return updated;
     }
 
     get profile(): ExperimentProfile {
         if (!this.profileField) {
-            throw Error('Experiment profile not initialized');
+            // throw Error('Experiment profile not initialized');
+            // set initProfile to prevent page broken
+            const initProfile = {
+                data: {
+                    "id": "", "revision": 0, "execDuration": 0,
+                    "logDir": "", "nextSequenceId": 0,
+                    "params": {
+                        "authorName": "", "experimentName": "", "trialConcurrency": 0, "maxExecDuration": 0, "maxTrialNum": 0, "searchSpace": "null",
+                        "trainingServicePlatform": "", "tuner": {
+                            "builtinTunerName": "TPE",
+                            "classArgs": { "optimize_mode": "" }, "checkpointDir": ""
+                        },
+                        "versionCheck": true, "clusterMetaData": [{ "key": "", "value": "" },
+                        { "key": "", "value": "" }]
+                    }, "startTime": 0, "endTime": 0
+                }
+            };
+            this.profileField = initProfile.data as any;
         }
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         return this.profileField!;
@@ -68,7 +124,9 @@ class Experiment {
 
     get status(): string {
         if (!this.statusField) {
-            throw Error('Experiment status not initialized');
+            // throw Error('Experiment status not initialized');
+            // this.statusField.status = '';
+            return '';
         }
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         return this.statusField!.status;

src/webui/src/static/model/trialmanager.ts

@@ -2,6 +2,7 @@ import axios from 'axios';
 import { MANAGER_IP, METRIC_GROUP_UPDATE_THRESHOLD, METRIC_GROUP_UPDATE_SIZE } from '../const';
 import { MetricDataRecord, TableRecord, TrialJobInfo } from '../interface';
 import { Trial } from './trial';
+import { requestAxios } from '../function';
 
 function groupMetricsByTrial(metrics: MetricDataRecord[]): Map<string, MetricDataRecord[]> {
     const ret = new Map<string, MetricDataRecord[]>();
@@ -39,9 +40,20 @@ class TrialManager {
     private maxSequenceId: number = 0;
     private doingBatchUpdate: boolean = false;
     private batchUpdatedAfterReading: boolean = false;
+    private isJobListError: boolean = false; // trial-jobs api error filed
+    private jobErrorMessage: string = ''; // trial-jobs error message
+    private isMetricdataError: boolean = false; // metric-data api error filed
+    private MetricdataErrorMessage: string = ''; // metric-data error message
+    private isLatestMetricdataError: boolean = false; // metric-data-latest api error filed
+    private latestMetricdataErrorMessage: string = ''; // metric-data-latest error message
+    private isMetricdataRangeError: boolean = false; // metric-data-range api error filed
+    private metricdataRangeErrorMessage: string = ''; // metric-data-latest error message
 
     public async init(): Promise<void> {
         while (!this.infoInitialized || !this.metricInitialized) {
+            if (this.isMetricdataError) {
+                return;
+            }
             await this.update();
         }
     }
@@ -156,24 +168,70 @@ class TrialManager {
         return trials;
     }
 
+    // if this.jobListError = true, show trial error message [/trial-jobs]
+    public jobListError(): boolean {
+        return this.isJobListError;
+    }
+
+    // trial error message's content [/trial-jobs]
+    public getJobErrorMessage(): string {
+        return this.jobErrorMessage;
+    }
+
+    // [/metric-data]
+    public MetricDataError(): boolean {
+        return this.isMetricdataError;
+    }
+
+    // [/metric-data]
+    public getMetricDataErrorMessage(): string {
+        return this.MetricdataErrorMessage;
+    }
+
+    // [/metric-data-latest]
+    public latestMetricDataError(): boolean {
+        return this.isLatestMetricdataError;
+    }
+
+    // [/metric-data-latest]
+    public getLatestMetricDataErrorMessage(): string {
+        return this.latestMetricdataErrorMessage;
+    }
+
+    public metricDataRangeError(): boolean {
+        return this.isMetricdataRangeError;
+    }
+
+    public metricDataRangeErrorMessage(): string {
+        return this.metricdataRangeErrorMessage;
+    }
+
     private async updateInfo(): Promise<boolean> {
-        const response = await axios.get(`${MANAGER_IP}/trial-jobs`);
+
         let updated = false;
-        if (response.status === 200) {
-            const newTrials = TrialManager.expandJobsToTrials(response.data);
-            for (const trialInfo of newTrials as TrialJobInfo[]) {
-                if (this.trials.has(trialInfo.id)) {
-                    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-                    updated = this.trials.get(trialInfo.id)!.updateTrialJobInfo(trialInfo) || updated;
-                } else {
-                    this.trials.set(trialInfo.id, new Trial(trialInfo, undefined));
-                    updated = true;
+        requestAxios(`${MANAGER_IP}/trial-jobs`)
+            .then(data => {
+                const newTrials = TrialManager.expandJobsToTrials(data as any);
+                for (const trialInfo of newTrials as TrialJobInfo[]) {
+                    if (this.trials.has(trialInfo.id)) {
+                        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                        updated = this.trials.get(trialInfo.id)!.updateTrialJobInfo(trialInfo) || updated;
+                    } else {
+                        this.trials.set(trialInfo.id, new Trial(trialInfo, undefined));
+                        updated = true;
+                    }
+                    this.maxSequenceId = Math.max(this.maxSequenceId, trialInfo.sequenceId);
                 }
-                this.maxSequenceId = Math.max(this.maxSequenceId, trialInfo.sequenceId);
-            }
-            this.infoInitialized = true;
-        }
-        return updated;
+                this.infoInitialized = true;
+            })
+            .catch(error => {
+                this.isJobListError = true;
+                this.jobErrorMessage = error.message;
+                this.infoInitialized = true;
+                updated = true;
+            });
+
+            return updated;
     }
 
     private async updateMetrics(lastTime?: boolean): Promise<boolean> {
@@ -188,13 +246,25 @@ class TrialManager {
     }
 
     private async updateAllMetrics(): Promise<boolean> {
-        const response = await axios.get(`${MANAGER_IP}/metric-data`);
-        return (response.status === 200) && this.doUpdateMetrics(response.data as MetricDataRecord[], false);
+        return requestAxios(`${MANAGER_IP}/metric-data`)
+            .then(data => this.doUpdateMetrics(data as any, false))
+            .catch(error => {
+                this.isMetricdataError = true;
+                this.MetricdataErrorMessage = `${error.message}`;
+                this.doUpdateMetrics([], false);
+                return true;
+            });
     }
 
     private async updateLatestMetrics(): Promise<boolean> {
-        const response = await axios.get(`${MANAGER_IP}/metric-data-latest`);
-        return (response.status === 200) && this.doUpdateMetrics(response.data as MetricDataRecord[], true);
+        return requestAxios(`${MANAGER_IP}/metric-data-latest`)
+            .then(data => this.doUpdateMetrics(data as any, true))
+            .catch(error => {
+                this.isLatestMetricdataError = true;
+                this.latestMetricdataErrorMessage = `${error.message}`;
+                this.doUpdateMetrics([], true);
+                return true;
+            });
     }
 
     private async updateManyMetrics(): Promise<void> {
@@ -202,12 +272,16 @@ class TrialManager {
             return;
         }
         this.doingBatchUpdate = true;
-        for (let i = 0; i < this.maxSequenceId; i += METRIC_GROUP_UPDATE_SIZE) {
-            const response = await axios.get(`${MANAGER_IP}/metric-data-range/${i}/${i + METRIC_GROUP_UPDATE_SIZE}`);
-            if (response.status === 200) {
-                const updated = this.doUpdateMetrics(response.data as MetricDataRecord[], false);
-                this.batchUpdatedAfterReading = this.batchUpdatedAfterReading || updated;
-            }
+        for (let i = 0; i < this.maxSequenceId && this.isMetricdataRangeError === false; i += METRIC_GROUP_UPDATE_SIZE) {
+            requestAxios(`${MANAGER_IP}/metric-data-range/${i}/${i + METRIC_GROUP_UPDATE_SIZE}`)
+                .then(data => {
+                    const updated = this.doUpdateMetrics(data as any, false);
+                    this.batchUpdatedAfterReading = this.batchUpdatedAfterReading || updated;
+                })
+                .catch(error => {
+                    this.isMetricdataRangeError = true;
+                    this.metricdataRangeErrorMessage = `${error.message}`;
+                });
         }
         this.doingBatchUpdate = false;
     }

src/webui/src/static/style/progress.scss

@@ -46,6 +46,9 @@
     .description{
       line-height: 34px;
       margin-left: 6px;
+      white-space: nowrap;
+      overflow: hidden;
+      text-overflow: ellipsis;
     }
   }