Allow NaN in metrics data (#1958)

src/nni_manager/core/nniDataStore.ts

@@ -4,6 +4,7 @@
 'use strict';
 
 import * as assert from 'assert';
+import * as JSON5 from 'json5';
 import { Deferred } from 'ts-deferred';
 
 import * as component from '../common/component';
@@ -131,7 +132,7 @@ class NNIDataStore implements DataStore {
     }
 
     public async storeMetricData(trialJobId: string, data: string): Promise<void> {
-        const metrics: MetricData = JSON.parse(data);
+        const metrics: MetricData = JSON5.parse(data);
         // REQUEST_PARAMETER is used to request new parameters for multiphase trial job,
         // it is not metrics, so it is skipped here.
         if (metrics.type === 'REQUEST_PARAMETER') {
@@ -140,7 +141,7 @@ class NNIDataStore implements DataStore {
         }
         assert(trialJobId === metrics.trial_job_id);
         try {
-            await this.db.storeMetricData(trialJobId, JSON.stringify({
+            await this.db.storeMetricData(trialJobId, JSON5.stringify({
                 trialJobId: metrics.trial_job_id,
                 parameterId: metrics.parameter_id,
                 type: metrics.type,

src/nni_manager/core/sqlDatabase.ts

@@ -5,6 +5,7 @@
 
 import * as assert from 'assert';
 import * as fs from 'fs';
+import * as JSON5 from 'json5';
 import * as path from 'path';
 import * as sqlite3 from 'sqlite3';
 import { Deferred } from 'ts-deferred';
@@ -202,10 +203,10 @@ class SqlDB implements Database {
 
     public storeMetricData(trialJobId: string, data: string): Promise<void> {
         const sql: string = 'insert into MetricData values (?,?,?,?,?,?)';
-        const json: MetricDataRecord = JSON.parse(data);
-        const args: any[] = [Date.now(), json.trialJobId, json.parameterId, json.type, json.sequence, JSON.stringify(json.data)];
+        const json: MetricDataRecord = JSON5.parse(data);
+        const args: any[] = [Date.now(), json.trialJobId, json.parameterId, json.type, json.sequence, JSON5.stringify(json.data)];
 
-        this.log.trace(`storeMetricData: SQL: ${sql}, args: ${JSON.stringify(args)}`);
+        this.log.trace(`storeMetricData: SQL: ${sql}, args: ${JSON5.stringify(args)}`);
         const deferred: Deferred<void> = new Deferred<void>();
         this.db.run(sql, args, (err: Error | null) => { this.resolve(deferred, err); });
 

src/nni_manager/package.json

@@ -16,6 +16,7 @@
     "express": "^4.16.3",
     "express-joi-validator": "^2.0.0",
     "js-base64": "^2.4.9",
+    "json5": "^2.1.1",
     "kubernetes-client": "^6.5.0",
     "rx": "^4.1.0",
     "sqlite3": "^4.0.2",
@@ -34,6 +35,7 @@
     "@types/express": "^4.16.0",
     "@types/glob": "^7.1.1",
     "@types/js-base64": "^2.3.1",
+    "@types/json5": "^0.0.30",
     "@types/mocha": "^5.2.5",
     "@types/node": "10.12.18",
     "@types/request": "^2.47.1",

src/nni_manager/yarn.lock

@@ -157,6 +157,10 @@
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.3.tgz#bdfd69d61e464dcc81b25159c270d75a73c1a636"
 
+"@types/json5@^0.0.30":
+  version "0.0.30"
+  resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.30.tgz#44cb52f32a809734ca562e685c6473b5754a7818"
+
 "@types/mime@*":
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/@types/mime/-/mime-2.0.0.tgz#5a7306e367c539b9f6543499de8dd519fac37a8b"
@@ -1840,9 +1844,9 @@ growl@1.10.5:
   version "1.10.5"
   resolved "https://registry.yarnpkg.com/growl/-/growl-1.10.5.tgz#f2735dc2283674fa67478b10181059355c369e5e"
 
-handlebars@^4.0.11, handlebars@^4.3.0:
-  version "4.5.3"
-  resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.5.3.tgz#5cf75bd8714f7605713511a56be7c349becb0482"
+handlebars@^4.0.11, handlebars@^4.5.3:
+  version "4.7.2"
+  resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.2.tgz#01127b3840156a0927058779482031afe0e730d7"
   dependencies:
     neo-async "^2.6.0"
     optimist "^0.6.1"
@@ -2371,6 +2375,12 @@ json-stringify-safe@~5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
 
+json5@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/json5/-/json5-2.1.1.tgz#81b6cb04e9ba496f1c7005d07b4368a2638f90b6"
+  dependencies:
+    minimist "^1.2.0"
+
 jsonparse@^1.2.0:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280"

src/sdk/pynni/nni/msg_dispatcher.py

@@ -11,7 +11,7 @@ from .msg_dispatcher_base import MsgDispatcherBase
 from .assessor import AssessResult
 from .common import multi_thread_enabled, multi_phase_enabled
 from .env_vars import dispatcher_env_vars
-from .utils import MetricType
+from .utils import MetricType, to_json
 
 _logger = logging.getLogger(__name__)
 
@@ -62,7 +62,7 @@ def _pack_parameter(parameter_id, params, customized=False, trial_job_id=None, p
         ret['parameter_index'] = parameter_index
     else:
         ret['parameter_index'] = 0
-    return json_tricks.dumps(ret)
+    return to_json(ret)
 
 
 class MsgDispatcher(MsgDispatcherBase):

src/sdk/pynni/nni/platform/local.py

@@ -6,10 +6,10 @@ import sys
 import json
 import time
 import subprocess
-import json_tricks
 
 from ..common import init_logger
 from ..env_vars import trial_env_vars
+from ..utils import to_json
 
 _sysdir = trial_env_vars.NNI_SYS_DIR
 if not os.path.exists(os.path.join(_sysdir, '.nni')):
@@ -30,7 +30,7 @@ _multiphase = trial_env_vars.MULTI_PHASE
 _param_index = 0
 
 def request_next_parameter():
-    metric = json_tricks.dumps({
+    metric = to_json({
         'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID,
         'type': 'REQUEST_PARAMETER',
         'sequence': 0,

src/sdk/pynni/nni/trial.py

 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
-import json_tricks
-
+from .utils import to_json
 from .env_vars import trial_env_vars
 from . import platform
 
@@ -110,7 +109,7 @@ def report_intermediate_result(metric):
     global _intermediate_seq
     assert _params or trial_env_vars.NNI_PLATFORM is None, \
         'nni.get_next_parameter() needs to be called before report_intermediate_result'
-    metric = json_tricks.dumps({
+    metric = to_json({
         'parameter_id': _params['parameter_id'] if _params else None,
         'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID,
         'type': 'PERIODICAL',
@@ -120,7 +119,6 @@ def report_intermediate_result(metric):
     _intermediate_seq += 1
     platform.send_metric(metric)
 
-
 def report_final_result(metric):
     """
     Reports final result to NNI.
@@ -132,7 +130,7 @@ def report_final_result(metric):
     """
     assert _params or trial_env_vars.NNI_PLATFORM is None, \
         'nni.get_next_parameter() needs to be called before report_final_result'
-    metric = json_tricks.dumps({
+    metric = to_json({
         'parameter_id': _params['parameter_id'] if _params else None,
         'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID,
         'type': 'FINAL',

src/sdk/pynni/nni/utils.py

 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT license.
 
-"""
-utils.py
-"""
-
 import os
+import functools
 from enum import Enum, unique
+import json_tricks
 
 from .common import init_logger
 from .env_vars import dispatcher_env_vars
 
+to_json = functools.partial(json_tricks.dumps, allow_nan=True)
+
 @unique
 class OptimizeMode(Enum):
     """Optimize Mode class

src/webui/package.json

@@ -18,6 +18,7 @@
     "fork-ts-checker-webpack-plugin": "^1.5.0",
     "fs-extra": "^8.1.0",
     "html-webpack-plugin": "^4.0.0-beta.8",
+    "json5": "^2.1.1",
     "less": "^3.9.0",
     "less-loader": "^5.0.0",
     "mini-css-extract-plugin": "^0.8.0",
@@ -55,6 +56,7 @@
     "eslint": "npx eslint ./ --ext .tsx,.ts"
   },
   "devDependencies": {
+    "@types/json5": "^0.0.30",
     "@types/node": "^10.14.14",
     "@types/react": "16.4.17",
     "@types/react-dom": "^16.0.7",

src/webui/src/components/trial-detail/TableList.tsx

@@ -6,7 +6,7 @@ import { ColumnProps } from 'antd/lib/table';
 const Option = Select.Option;
 const CheckboxGroup = Checkbox.Group;
 import { MANAGER_IP, trialJobStatus, COLUMN_INDEX, COLUMNPro } from '../../static/const';
-import { convertDuration, formatTimestamp, intermediateGraphOption, killJob } from '../../static/function';
+import { convertDuration, formatTimestamp, intermediateGraphOption, killJob, parseMetrics } from '../../static/function';
 import { EXPERIMENT, TRIALS } from '../../static/datamodel';
 import { TableRecord } from '../../static/interface';
 import OpenRow from '../public-child/OpenRow';
@@ -178,11 +178,11 @@ class TableList extends React.Component<TableListProps, TableListState> {
             // get intermediate result dict keys array
             let otherkeys: Array<string> = ['default'];
             if (res.data.length !== 0) {
-                otherkeys = Object.keys(JSON.parse(res.data[0].data));
+                otherkeys = Object.keys(parseMetrics(res.data[0].data));
             }
             // intermediateArr just store default val
             Object.keys(res.data).map(item => {
-                const temp = JSON.parse(res.data[item].data);
+                const temp = parseMetrics(res.data[item].data);
                 if (typeof temp === 'object') {
                     intermediateArr.push(temp.default);
                 } else {
@@ -210,7 +210,7 @@ class TableList extends React.Component<TableListProps, TableListState> {
         // just watch default key-val
         if (isShowDefault === true) {
             Object.keys(intermediateData).map(item => {
-                const temp = JSON.parse(intermediateData[item].data);
+                const temp = parseMetrics(intermediateData[item].data);
                 if (typeof temp === 'object') {
                     intermediateArr.push(temp[value]);
                 } else {
@@ -219,7 +219,7 @@ class TableList extends React.Component<TableListProps, TableListState> {
             });
         } else {
             Object.keys(intermediateData).map(item => {
-                const temp = JSON.parse(intermediateData[item].data);
+                const temp = parseMetrics(intermediateData[item].data);
                 if (typeof temp === 'object') {
                     intermediateArr.push(temp[value]);
                 }

src/webui/src/static/function.ts

+import * as JSON5 from 'json5';
 import axios from 'axios';
 import { message } from 'antd';
 import { MANAGER_IP } from './const';
@@ -173,8 +174,16 @@ function formatTimestamp(timestamp?: number, placeholder?: string = 'N/A'): stri
     return timestamp ? new Date(timestamp).toLocaleString('en-US') : placeholder;
 }
 
+function parseMetrics(metricData: string): any {
+    if (metricData.includes('NaN')) {
+        return JSON5.parse(metricData)
+    } else {
+        return JSON.parse(metricData)
+    }
+}
+
 function metricAccuracy(metric: MetricDataRecord): number {
-    const data = JSON.parse(metric.data);
+    const data = parseMetrics(metric.data);
     return typeof data === 'number' ? data : NaN;
 }
 
@@ -186,5 +195,5 @@ function formatAccuracy(accuracy: number): string {
 export {
     convertTime, convertDuration, getFinalResult, getFinal, downFile,
     intermediateGraphOption, killJob, filterByStatus, filterDuration,
-    formatAccuracy, formatTimestamp, metricAccuracy
+    formatAccuracy, formatTimestamp, metricAccuracy, parseMetrics
 };

src/webui/src/static/model/trial.ts

 import { MetricDataRecord, TrialJobInfo, TableObj, TableRecord, Parameters, FinalType } from '../interface';
-import { getFinal, formatAccuracy, metricAccuracy } from '../function';
+import { getFinal, formatAccuracy, metricAccuracy, parseMetrics } from '../function';
 
 class Trial implements TableObj {
     private metricsInitialized: boolean = false;
@@ -56,7 +56,7 @@ class Trial implements TableObj {
             // TODO: support intermeidate result is dict
             const temp = this.intermediates[this.intermediates.length - 1];
             if (temp !== undefined) {
-                return JSON.parse(temp.data);
+                return parseMetrics(temp.data);
             } else {
                 return undefined;
             }
@@ -138,10 +138,10 @@ class Trial implements TableObj {
 
         const mediate: number[] = [ ];
         for (const items of this.intermediateMetrics) {
-            if (typeof JSON.parse(items.data) === 'object') {
-                mediate.push(JSON.parse(items.data).default);
+            if (typeof parseMetrics(items.data) === 'object') {
+                mediate.push(parseMetrics(items.data).default);
             } else {
-                mediate.push(JSON.parse(items.data));
+                mediate.push(parseMetrics(items.data));
             }
         }
         ret.intermediate = mediate;

src/webui/yarn.lock

@@ -80,6 +80,10 @@
   version "0.0.29"
   resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.29.tgz#ee28707ae94e11d2b827bcbe5270bcea7f3e71ee"
 
+"@types/json5@^0.0.30":
+  version "0.0.30"
+  resolved "https://registry.yarnpkg.com/@types/json5/-/json5-0.0.30.tgz#44cb52f32a809734ca562e685c6473b5754a7818"
+
 "@types/minimatch@*":
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
@@ -3988,6 +3992,12 @@ json5@^1.0.1:
   dependencies:
     minimist "^1.2.0"
 
+json5@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/json5/-/json5-2.1.1.tgz#81b6cb04e9ba496f1c7005d07b4368a2638f90b6"
+  dependencies:
+    minimist "^1.2.0"
+
 jsonfile@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb"