deal with all type metrics (#2782)

* update

* update

* Fix metrics for type other than number

* update

* change master version TableList.tsx

* revert unuseful change
Co-authored-by: Lijiao <Lijiaoa@outlook.com>
Co-authored-by: Lijiao <1425861283@qq.com>
Co-authored-by: Yuge Zhang <scottyugochang@gmail.com>

src/webui/src/App.tsx

@@ -15,13 +15,13 @@ interface AppState {
     isillegalFinal: boolean;
     expWarningMessage: string;
     bestTrialEntries: string; // for overview page: best trial entreis
+    isUpdate: boolean;
 }
 
 class App extends React.Component<{}, AppState> {
     private timerId!: number | undefined;
     private dataFormatimer!: number;
     private firstLoad: boolean = false; // when click refresh selector options
-    
     constructor(props: {}) {
         super(props);
         this.state = {
@@ -32,16 +32,19 @@ class App extends React.Component<{}, AppState> {
             metricGraphMode: 'max',
             isillegalFinal: false,
             expWarningMessage: '',
-            bestTrialEntries: '10'
+            bestTrialEntries: '10',
+            isUpdate: true
         };
     }
 
     async componentDidMount(): Promise<void> {
         await Promise.all([EXPERIMENT.init(), TRIALS.init()]);
-        this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1 }));
-        this.setState(state => ({ trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 }));
-        this.timerId = window.setTimeout(this.refresh, this.state.interval * 1000);
-        this.setState({ metricGraphMode: (EXPERIMENT.optimizeMode === 'minimize' ? 'min' : 'max') });
+        this.setState(state => ({ 
+            experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1, 
+            trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1,
+            metricGraphMode: (EXPERIMENT.optimizeMode === 'minimize' ? 'min' : 'max')
+        }));
+        this.timerId = window.setTimeout(this.refresh, this.state.interval * 100);
         // final result is legal
         // get a succeed trial，see final result data's format
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
@@ -99,6 +102,15 @@ class App extends React.Component<{}, AppState> {
         this.setState({ bestTrialEntries: entries });
     }
 
+    shouldComponentUpdate(nextProps: any, nextState: AppState): boolean {
+        
+        if(!(nextState.isUpdate || nextState.isUpdate === undefined)){
+            nextState.isUpdate = true;
+            return false;
+        }
+        return true;
+    }
+
     render(): React.ReactNode {
         const { interval, columnList, experimentUpdateBroadcast, trialsUpdateBroadcast,
             metricGraphMode, isillegalFinal, expWarningMessage, bestTrialEntries
@@ -106,7 +118,6 @@ class App extends React.Component<{}, AppState> {
         if (experimentUpdateBroadcast === 0 || trialsUpdateBroadcast === 0) {
             return null;  // TODO: render a loading page
         }
-        
         const errorList = [
             { errorWhere: TRIALS.jobListError(), errorMessage: TRIALS.getJobErrorMessage() },
             { errorWhere: EXPERIMENT.experimentError(), errorMessage: EXPERIMENT.getExperimentMessage() },
@@ -158,7 +169,6 @@ class App extends React.Component<{}, AppState> {
     }
 
     private refresh = async (): Promise<void> => {
-
         // resolve this question: 10s -> 20s, page refresh twice.
         // only refresh this page after clicking the refresh options
         if (this.firstLoad !== true) {
@@ -177,8 +187,7 @@ class App extends React.Component<{}, AppState> {
         // experiment status and /trial-jobs api's status could decide website update
         if (['DONE', 'ERROR', 'STOPPED'].includes(EXPERIMENT.status) || TRIALS.jobListError()) {
             // experiment finished, refresh once more to ensure consistency
-            this.setState({ interval: 0 });
-            this.lastRefresh();
+            this.setState(() => ({ interval: 0, isUpdate: false }));
             return;
         }
 
@@ -189,8 +198,7 @@ class App extends React.Component<{}, AppState> {
     public async lastRefresh(): Promise<void> {
         await EXPERIMENT.update();
         await TRIALS.update(true);
-        this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1 }));
-        this.setState(state => ({ trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 }));
+        this.setState(state => ({ experimentUpdateBroadcast: state.experimentUpdateBroadcast + 1, trialsUpdateBroadcast: state.trialsUpdateBroadcast + 1 }));
     }
 }
 

src/webui/src/components/trial-detail/Para.tsx

@@ -195,7 +195,7 @@ class Para extends React.Component<ParaProps, ParaState> {
             }]);
         }
 
-        if (convertedTrials.length === 0) {
+        if (convertedTrials.length === 0 || dimensions.length <= 1) {
             return;
         }
 

src/webui/src/static/function.ts

@@ -51,7 +51,7 @@ const convertDuration = (num: number): string => {
 };
 
 function parseMetrics(metricData: string): any {
-    if (metricData.includes('NaN')) {
+    if (metricData.includes('NaN') || metricData.includes('Infinity')) {
         return JSON5.parse(JSON5.parse(metricData));
     } else {
         return JSON.parse(JSON.parse(metricData));
@@ -84,15 +84,18 @@ const getFinalResult = (final?: MetricDataRecord[]): number => {
     }
 };
 
+function isNaNorInfinity(val: number): boolean {
+    return Object.is(val, NaN) || Object.is(val, Infinity);
+}
+
 // get final result value // acc obj
 const getFinal = (final?: MetricDataRecord[]): FinalType | undefined => {
     let showDefault: FinalType;
     if (final) {
         showDefault = parseMetrics(final[final.length - 1].data);
         if (typeof showDefault === 'number') {
-            if(!isNaN(showDefault)){
-                showDefault = { default: showDefault };
-                return showDefault;
+            if(!isNaNorInfinity(showDefault)){
+                return { default: showDefault };
             }
         } else if (isArrayType(showDefault)) {
             // not support final type
@@ -165,11 +168,9 @@ const killJob = (key: number, id: string, status: string, updateList?: Function)
         .catch(error => {
             if (error.response.status === 500) {
                 if (error.response.data.error) {
-                    alert(123);
-                    // message.error(error.response.data.error);
+                    alert(error.response.data.error);
                 } else {
-                    alert(234);
-                    // message.error('500 error, fail to cancel the job');
+                    alert('500 error, fail to cancel the job');
                 }
             }
         });
@@ -229,9 +230,17 @@ function formatAccuracy(accuracy: number): string {
     return accuracy.toFixed(6).replace(/0+$/, '').replace(/\.$/, '');
 }
 
+function formatComplexTypeValue(value: any): string | number {
+    if (['number', 'string'].includes(typeof value)) {
+        return value;
+    } else {
+        return value.toString();
+    }
+}
+
 export {
     convertTime, convertDuration, getFinalResult, getFinal, downFile,
     intermediateGraphOption, killJob, filterByStatus, filterDuration,
     formatAccuracy, formatTimestamp, metricAccuracy, parseMetrics,
-    isArrayType, requestAxios
+    isArrayType, requestAxios, isNaNorInfinity, formatComplexTypeValue
 };

src/webui/src/static/interface.ts

@@ -46,7 +46,7 @@ interface TableRecord {
     duration: number;
     status: string;
     intermediateCount: number;
-    accuracy?: number;
+    accuracy?: number | any;
     latestAccuracy: number | undefined;
     formattedLatestAccuracy: string; // format (LATEST/FINAL),
     accDictionary: FinalType | undefined;

src/webui/src/static/model/experiment.ts

@@ -58,7 +58,7 @@ class Experiment {
 
         await requestAxios(`${MANAGER_IP}/experiment`)
             .then(data => {
-                updated = updated || compareProfiles(this.profileField, data);
+                updated = updated || !compareProfiles(this.profileField, data);
                 this.profileField = data;
             })
             .catch(error => {
@@ -69,7 +69,7 @@ class Experiment {
 
         await requestAxios(`${MANAGER_IP}/check-status`)
             .then(data => {
-                updated = JSON.stringify(this.statusField) === JSON.stringify(data);
+                updated = JSON.stringify(this.statusField) !== JSON.stringify(data);
                 this.statusField = data;
             })
             .catch(error => {

src/webui/src/static/model/searchspace.ts

 import { SingleAxis, MultipleAxes, TableObj } from '../interface';
 import { SUPPORTED_SEARCH_SPACE_TYPE } from '../const';
+import { formatComplexTypeValue } from '../function';
 
 function fullNameJoin(prefix: string, name: string): string {
     return prefix ? (prefix + '/' + name) : name;
@@ -52,7 +53,7 @@ class SimpleOrdinalAxis implements SingleAxis {
         this.baseName = baseName;
         this.fullName = fullName;
         this.type = type;
-        this.domain = value;
+        this.domain = Array.from(value).map(formatComplexTypeValue);
     }
 }
 
@@ -115,7 +116,7 @@ export class SearchSpace implements MultipleAxes {
                 trial.parameters(searchSpace);
             } catch (unexpectedEntries) {
                 // eslint-disable-next-line no-console
-                console.log(unexpectedEntries);
+                console.warn(unexpectedEntries);
                 for (const [k, v] of unexpectedEntries as Map<string, any>) {
                     const column = addingColumns.get(k);
                     if (column === undefined) {
@@ -164,7 +165,7 @@ export class MetricSpace implements MultipleAxes {
             if (value.every(v => typeof v === 'number')) {
                 this.axes.set(key, new NumericAxis(key, key, 'uniform', [Math.min(...value), Math.max(...value)]));
             } else {
-                // TODO: skip for now
+                this.axes.set(key, new SimpleOrdinalAxis(key, key, 'choice', value));
             }
         });
     }

src/webui/src/static/model/trial.ts

+import * as JSON5 from 'json5';
 import { MetricDataRecord, TrialJobInfo, TableObj, TableRecord, Parameters, FinalType, MultipleAxes, SingleAxis } from '../interface';
-import { getFinal, formatAccuracy, metricAccuracy, parseMetrics, isArrayType } from '../function';
+import { getFinal, formatAccuracy, metricAccuracy, parseMetrics, isArrayType, isNaNorInfinity, formatComplexTypeValue } from '../function';
 
 /**
  * Get a structured representation of parameters
@@ -27,10 +28,10 @@ function inferTrialParameters(paramObj: object, space: MultipleAxes, prefix: str
                     subUnexpected.forEach((v, k) => unexpectedEntries.set(k, v));
                 }
             } else {
-                parameters.set(axisKey, v);
+                parameters.set(axisKey, formatComplexTypeValue(v));
             }
         } else {
-            unexpectedEntries.set(prefix + k, v);
+            unexpectedEntries.set(prefix + k, formatComplexTypeValue(v));
         }
     }
     return [parameters, unexpectedEntries];
@@ -110,7 +111,15 @@ class Trial implements TableObj {
         const endTime = this.info.endTime || new Date().getTime();
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         const duration = (endTime - this.info.startTime!) / 1000;
-
+        let accuracy;
+        if(this.acc !== undefined && this.acc.default !== undefined){
+            if(typeof this.acc.default === 'number'){
+                accuracy = JSON5.parse(this.acc.default);
+            }else {
+                accuracy = this.acc.default;
+            }
+        }
+        
         return {
             key: this.info.id,
             sequenceId: this.info.sequenceId,
@@ -121,8 +130,7 @@ class Trial implements TableObj {
             duration,
             status: this.info.status,
             intermediateCount: this.intermediates.length,
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            accuracy: this.acc !== undefined ? JSON.parse(this.acc!.default) : undefined,
+            accuracy: accuracy,
             latestAccuracy: this.latestAccuracy,
             formattedLatestAccuracy: this.formatLatestAccuracy(),
             accDictionary: this.acc
@@ -152,6 +160,9 @@ class Trial implements TableObj {
     }
 
     get acc(): FinalType | undefined {
+        if (this.info === undefined) {
+            return undefined;
+        }
         return getFinal(this.info.finalMetricData);
     }
 
@@ -190,10 +201,10 @@ class Trial implements TableObj {
     }
 
     public parameters(axes: MultipleAxes): Map<SingleAxis, any> {
-        const tempHyper = this.info.hyperParameters;
-        if (tempHyper === undefined) {
-            throw new Map([['error', 'This trial\'s parameters are not available.']]);
+        if (this.info === undefined || this.info.hyperParameters === undefined) {
+            throw new Map();
         } else {
+            const tempHyper = this.info.hyperParameters;
             let params = JSON.parse(tempHyper[tempHyper.length - 1]).parameters;
             if (typeof params === 'string') {
                 params = JSON.parse(params);
@@ -216,6 +227,7 @@ class Trial implements TableObj {
         Object.entries(acc).forEach(item => {
             const [k, v] = item;
             const column = space.axes.get(k);
+            
             if (column !== undefined) {
                 ret.set(column, v);
             } else {
@@ -233,8 +245,11 @@ class Trial implements TableObj {
     }
 
     public finalKeys(): string[] {
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        return Object.keys(this.acc!);
+        if(this.acc !== undefined){
+            return Object.keys(this.acc);
+        } else {
+            return [];
+        }
     }
 
     /* table obj end */
@@ -288,24 +303,34 @@ class Trial implements TableObj {
         return !same;
     }
 
-    public formatLatestAccuracy(): string {  // TODO: this should be private
-        if (this.accuracy !== undefined) {
-            if (isNaN(this.accuracy)) {
-                return this.accuracy.toString();
+    private renderNumber(val: any): string {
+        if(typeof val === 'number'){
+            if (isNaNorInfinity(val)) {
+                return `${val}`; // show 'NaN' or 'Infinity'
             } else {
-                return `${formatAccuracy(this.accuracy)} (FINAL)`;
+                return `${formatAccuracy(val)} (FINAL)`;
             }
-        } else if (this.intermediates.length === 0) {
-            return '--';
         } else {
-            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            const latest = this.intermediates[this.intermediates.length - 1]!;
-            if (isNaN(metricAccuracy(latest))) {
-                return 'NaN';
+            // show other types, such as {tensor: {data: }}
+            return JSON.stringify(val);
+        }
+    }
+
+    public formatLatestAccuracy(): string {  // TODO: this should be private
+        if(this.status === 'SUCCEEDED'){
+            return (this.accuracy === undefined ? '--': this.renderNumber(this.accuracy));
+        } else {
+            if (this.accuracy !== undefined) {
+                return this.renderNumber(this.accuracy);
+            } else if (this.intermediates.length === 0) {
+                return '--';
             } else {
-                return `${formatAccuracy(metricAccuracy(latest))} (LATEST)`;
+                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+                const latest = this.intermediates[this.intermediates.length - 1]!;
+                return this.renderNumber(metricAccuracy(latest));
             }
         }
+        
     }
 }