Show the curve for other keys in metric dict in the "default metric graph",...

Show the curve for other keys in metric dict in the "default metric graph", currently only support the default key (#4973)

ts/webui/src/components/experiment/trialdetail/TrialsDetail.tsx

@@ -48,8 +48,7 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
     render(): React.ReactNode {
         const { whichChart } = this.state;
         const source = TRIALS.toArray();
-        const trialIds = TRIALS.toArray().map(trial => trial.id);
-
+        const succeededTrialIds = TRIALS.succeededTrials().map(trial => trial.id);
         return (
             <AppContext.Consumer>
                 {(_value): React.ReactNode => (
@@ -65,7 +64,7 @@ class TrialsDetail extends React.Component<{}, TrialDetailState> {
                                 <PivotItem headerText='Default metric' itemIcon='HomeGroup' key='Default metric'>
                                     <Stack className='graph'>
                                         <DefaultPoint
-                                            trialIds={trialIds}
+                                            trialIds={succeededTrialIds}
                                             hasBestCurve={true}
                                             chartHeight={402}
                                             changeExpandRowIDs={_value.changeExpandRowIDs}

ts/webui/src/components/experiment/trialdetail/chart/DefaultMetricPoint.tsx

@@ -11,6 +11,8 @@ import 'echarts/lib/chart/scatter';
 import 'echarts/lib/component/tooltip';
 import 'echarts/lib/component/title';
 
+// this file is for overview page and detail page's Default metric graph
+
 const EmptyGraph = {
     grid: {
         left: '8%'
@@ -37,6 +39,7 @@ interface DefaultPointState {
     startY: number; // dataZoomY
     endY: number;
     userSelectOptimizeMode: string;
+    userSelectAccuracyNumberKey: string;
 }
 
 class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState> {
@@ -46,7 +49,8 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
             bestCurveEnabled: false,
             startY: 0, // dataZoomY
             endY: 100,
-            userSelectOptimizeMode: optimizeModeValue(EXPERIMENT.optimizeMode)
+            userSelectOptimizeMode: optimizeModeValue(EXPERIMENT.optimizeMode),
+            userSelectAccuracyNumberKey: 'default'
         };
     }
 
@@ -72,7 +76,7 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
     };
 
     generateGraphConfig(_maxSequenceId: number): any {
-        const { startY, endY } = this.state;
+        const { startY, endY, userSelectAccuracyNumberKey } = this.state;
         const { hasBestCurve } = this.props;
         return {
             grid: {
@@ -86,7 +90,7 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
                     <div class="tooldetailAccuracy">
                         <div>Trial No.: ${data.data[0]}</div>
                         <div>Trial ID: ${data.data[2]}</div>
-                        <div>Default metric: ${data.data[1]}</div>
+                        <div>${userSelectAccuracyNumberKey}: ${data.data[1]}</div>
                         <div>Parameters: <pre>${JSON.stringify(
                             reformatRetiariiParameter(data.data[3]),
                             null,
@@ -118,8 +122,32 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
         };
     }
 
+    private formatAccuracy(accuracy: number | undefined): number {
+        if (accuracy === undefined || isNaN(accuracy) || !isFinite(accuracy)) {
+            return 0;
+        }
+        return accuracy;
+    }
+
     generateScatterSeries(trials: Trial[]): any {
-        const data = trials.map(trial => [trial.sequenceId, trial.accuracy, trial.id, trial.description.parameters]);
+        const { userSelectAccuracyNumberKey } = this.state;
+        let data;
+        if (trials[0].accuracyNumberTypeDictKeys.length > 1) {
+            // dict type final results
+            data = trials.map(trial => [
+                trial.sequenceId,
+                trial.acc === undefined ? 0 : this.formatAccuracy(trial.acc[userSelectAccuracyNumberKey]),
+                trial.id,
+                trial.description.parameters
+            ]);
+        } else {
+            data = trials.map(trial => [
+                trial.sequenceId,
+                this.formatAccuracy(trial.accuracy),
+                trial.id,
+                trial.description.parameters
+            ]);
+        }
         return {
             symbolSize: 6,
             type: 'scatter',
@@ -128,20 +156,36 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
     }
 
     generateBestCurveSeries(trials: Trial[]): any {
-        const { userSelectOptimizeMode } = this.state;
+        const { userSelectOptimizeMode, userSelectAccuracyNumberKey } = this.state;
         let best = trials[0];
-        const data = [[best.sequenceId, best.accuracy, best.id, best.description.parameters]];
-
+        const data = [
+            [
+                best.sequenceId,
+                best.acc === undefined ? 0 : this.formatAccuracy(best.acc[userSelectAccuracyNumberKey]),
+                best.id,
+                best.description.parameters
+            ]
+        ];
         for (let i = 1; i < trials.length; i++) {
             const trial = trials[i];
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            const delta = trial.accuracy! - best.accuracy!;
+            const delta = trial.acc![userSelectAccuracyNumberKey] - best.acc![userSelectAccuracyNumberKey];
             const better = userSelectOptimizeMode === 'minimize' ? delta < 0 : delta > 0;
             if (better) {
-                data.push([trial.sequenceId, trial.accuracy, best.id, trial.description.parameters]);
+                data.push([
+                    trial.sequenceId,
+                    trial.acc === undefined ? 0 : this.formatAccuracy(trial.acc[userSelectAccuracyNumberKey]),
+                    best.id,
+                    trial.description.parameters
+                ]);
                 best = trial;
             } else {
-                data.push([trial.sequenceId, best.accuracy, best.id, trial.description.parameters]);
+                data.push([
+                    trial.sequenceId,
+                    best.acc === undefined ? 0 : this.formatAccuracy(best.acc[userSelectAccuracyNumberKey]),
+                    best.id,
+                    trial.description.parameters
+                ]);
             }
         }
 
@@ -152,20 +196,37 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
         };
     }
 
-    // get user mode number 'max' or 'min'
     updateUserOptimizeMode = (event: React.FormEvent<HTMLDivElement>, item?: IDropdownOption): void => {
         if (item !== undefined) {
             this.setState({ userSelectOptimizeMode: item.key.toString() });
         }
     };
 
+    // final result keys dropdown click event
+    updateTrialfinalResultKeys = (event: React.FormEvent<HTMLDivElement>, item?: IDropdownOption): void => {
+        if (item !== undefined) {
+            this.setState(
+                {
+                    userSelectAccuracyNumberKey: item.key.toString()
+                },
+                () => {
+                    this.generateGraph();
+                }
+            );
+        }
+    };
+
     render(): React.ReactNode {
         const { hasBestCurve, chartHeight } = this.props;
-        const { userSelectOptimizeMode } = this.state;
+        const { userSelectOptimizeMode, userSelectAccuracyNumberKey } = this.state;
+        const trials = TRIALS.getTrials(this.props.trialIds).filter(trial => trial.sortable);
         const graph = this.generateGraph();
         const accNodata = graph === EmptyGraph ? 'No data' : '';
         const onEvents = { dataZoom: this.metricDataZoom, click: this.pointClick };
-
+        let dictDropdown: string[] = [];
+        if (trials.length > 0) {
+            dictDropdown = trials[0].accuracyNumberTypeDictKeys;
+        }
         return (
             <div>
                 {hasBestCurve && (
@@ -181,6 +242,15 @@ class DefaultPoint extends React.Component<DefaultPointProps, DefaultPointState>
                             styles={{ dropdown: { width: 100 } }}
                             className='para-filter-percent'
                         />
+                        {dictDropdown.length > 1 && (
+                            <Dropdown
+                                selectedKey={userSelectAccuracyNumberKey}
+                                onChange={this.updateTrialfinalResultKeys}
+                                options={dictDropdown.map(item => ({ key: item, text: item }))}
+                                styles={{ dropdown: { width: 100 } }}
+                                className='para-filter-percent'
+                            />
+                        )}
                     </Stack>
                 )}
                 <div className='default-metric-graph graph'>

ts/webui/src/static/interface.ts

@@ -56,7 +56,6 @@ interface TableRecord {
     accuracy?: number | any;
     latestAccuracy: number | undefined;
     formattedLatestAccuracy: string; // format (LATEST/FINAL),
-    accDictionary: FinalType | undefined;
 }
 
 interface SearchSpace {

ts/webui/src/static/model/trial.ts

@@ -159,8 +159,7 @@ class Trial implements TableObj {
             intermediateCount: this.intermediates.length,
             accuracy: accuracy,
             latestAccuracy: this.latestAccuracy,
-            formattedLatestAccuracy: this.formatLatestAccuracy(),
-            accDictionary: this.acc
+            formattedLatestAccuracy: this.formatLatestAccuracy()
         };
     }
 
@@ -193,6 +192,22 @@ class Trial implements TableObj {
         return getFinal(this.info.finalMetricData);
     }
 
+    get accuracyNumberTypeDictKeys(): string[] {
+        let accuracyTypeList: string[] = [];
+
+        if (this.acc !== undefined) {
+            for (const [item, value] of Object.entries(this.acc)) {
+                if (typeof value === 'number') {
+                    accuracyTypeList.push(item);
+                }
+            }
+        } else {
+            accuracyTypeList = ['default'];
+        }
+
+        return accuracyTypeList;
+    }
+
     get description(): Parameters {
         const ret: Parameters = {
             parameters: {},
@@ -272,10 +287,6 @@ class Trial implements TableObj {
         return ret;
     }
 
-    get color(): string | undefined {
-        return undefined;
-    }
-
     public finalKeys(): string[] {
         if (this.acc !== undefined) {
             return Object.keys(this.acc);