Merge pull request #950 from Microsoft/v0.6

Merge V0.6 branch to master

src/sdk/pynni/nni/smac_tuner/smac_tuner.py

@@ -192,17 +192,20 @@ class SMACTuner(Tuner):
         Returns
         -------
         dict
-            challenger dict
+            dict which stores copy of challengers
         """
+        converted_dict = {}
         for key, value in challenger_dict.items():
             # convert to loguniform
             if key in self.loguniform_key:
-                challenger_dict[key] = np.exp(challenger_dict[key])
+                converted_dict[key] = np.exp(challenger_dict[key])
             # convert categorical back to original value
-            if key in self.categorical_dict:
+            elif key in self.categorical_dict:
                 idx = challenger_dict[key]
-                challenger_dict[key] = self.categorical_dict[key][idx]
-        return challenger_dict
+                converted_dict[key] = self.categorical_dict[key][idx]
+            else:
+                converted_dict[key] = value
+        return converted_dict
 
     def generate_parameters(self, parameter_id):
         """generate one instance of hyperparameters
@@ -220,13 +223,11 @@ class SMACTuner(Tuner):
         if self.first_one:
             init_challenger = self.smbo_solver.nni_smac_start()
             self.total_data[parameter_id] = init_challenger
-            json_tricks.dumps(init_challenger.get_dictionary())
             return self.convert_loguniform_categorical(init_challenger.get_dictionary())
         else:
             challengers = self.smbo_solver.nni_smac_request_challengers()
             for challenger in challengers:
                 self.total_data[parameter_id] = challenger
-                json_tricks.dumps(challenger.get_dictionary())
                 return self.convert_loguniform_categorical(challenger.get_dictionary())
 
     def generate_multiple_parameters(self, parameter_id_list):
@@ -247,7 +248,6 @@ class SMACTuner(Tuner):
             for one_id in parameter_id_list:
                 init_challenger = self.smbo_solver.nni_smac_start()
                 self.total_data[one_id] = init_challenger
-                json_tricks.dumps(init_challenger.get_dictionary())
                 params.append(self.convert_loguniform_categorical(init_challenger.get_dictionary()))
         else:
             challengers = self.smbo_solver.nni_smac_request_challengers()
@@ -257,7 +257,6 @@ class SMACTuner(Tuner):
                 if cnt >= len(parameter_id_list):
                     break
                 self.total_data[parameter_id_list[cnt]] = challenger
-                json_tricks.dumps(challenger.get_dictionary())
                 params.append(self.convert_loguniform_categorical(challenger.get_dictionary()))
                 cnt += 1
         return params

src/webui/src/components/Overview.tsx

@@ -20,6 +20,7 @@ require('../static/style/overviewTitle.scss');
 
 interface OverviewState {
     tableData: Array<TableObj>;
+    experimentAPI: object;
     searchSpace: object;
     status: string;
     errorStr: string;
@@ -47,6 +48,7 @@ class Overview extends React.Component<{}, OverviewState> {
         super(props);
         this.state = {
             searchSpace: {},
+            experimentAPI: {},
             status: '',
             errorStr: '',
             trialProfile: {
@@ -143,6 +145,7 @@ class Overview extends React.Component<{}, OverviewState> {
                     });
                     if (this._isMounted) {
                         this.setState({
+                            experimentAPI: res.data,
                             trialProfile: trialPro[0],
                             searchSpace: searchSpace,
                             isLogCollection: expLogCollection
@@ -390,7 +393,7 @@ class Overview extends React.Component<{}, OverviewState> {
         const {
             trialProfile, searchSpace, tableData, accuracyData,
             accNodata, status, errorStr, trialNumber, bestAccuracy,
-            titleMaxbgcolor, titleMinbgcolor, isLogCollection
+            titleMaxbgcolor, titleMinbgcolor, isLogCollection, experimentAPI
         } = this.state;
 
         return (
@@ -425,9 +428,7 @@ class Overview extends React.Component<{}, OverviewState> {
                         <Row className="experiment">
                             {/* the scroll bar all the trial profile in the searchSpace div*/}
                             <div className="experiment searchSpace">
-                                <TrialPro
-                                    tiralProInfo={trialProfile}
-                                />
+                            <TrialPro experiment={experimentAPI} />
                             </div>
                         </Row>
                     </Col>

src/webui/src/components/overview/TrialProfile.tsx

 import * as React from 'react';
-import { Experiment } from '../../static/interface';
 import MonacoEditor from 'react-monaco-editor';
 import { MONACO } from '../../static/const';
 
 interface TrialInfoProps {
-    tiralProInfo: Experiment;
+    experiment: object;
 }
 
 class TrialInfo extends React.Component<TrialInfoProps, {}> {
@@ -13,19 +12,32 @@ class TrialInfo extends React.Component<TrialInfoProps, {}> {
         super(props);
     }
 
-    render() {
-        const { tiralProInfo } = this.props;
-        const showProInfo = [];
-        showProInfo.push({
-            revision: tiralProInfo.revision,
-            authorName: tiralProInfo.author,
-            trialConcurrency: tiralProInfo.runConcurren,
-            tuner: tiralProInfo.tuner,
-            assessor: tiralProInfo.assessor ? tiralProInfo.assessor : undefined,
-            logCollection: tiralProInfo.logCollection ? tiralProInfo.logCollection : undefined,
-            advisor: tiralProInfo.advisor ? tiralProInfo.advisor : undefined,
-            clusterMetaData: tiralProInfo.clusterMetaData ? tiralProInfo.clusterMetaData : undefined
+    componentWillReceiveProps(nextProps: TrialInfoProps) {
+        const experiments = nextProps.experiment;
+        Object.keys(experiments).map(key => {
+            switch (key) {
+                case 'id':
+                case 'logDir':
+                case 'startTime':
+                case 'endTime':
+                    experiments[key] = undefined;
+                    break;
+                case 'params':
+                    const params = experiments[key];
+                    Object.keys(params).map(item => {
+                        if (item === 'experimentName' || item === 'searchSpace'
+                            || item === 'trainingServicePlatform') {
+                            params[item] = undefined;
+                        }
+                    });
+                    break;
+                default:
+            }
         });
+    }
+
+    render() {
+        const { experiment } = this.props;
         return (
             <div className="profile">
                 <MonacoEditor
@@ -33,7 +45,7 @@ class TrialInfo extends React.Component<TrialInfoProps, {}> {
                     height="380"
                     language="json"
                     theme="vs-light"
-                    value={JSON.stringify(showProInfo[0], null, 2)}
+                    value={JSON.stringify(experiment, null, 2)}
                     options={MONACO}
                 />
             </div>

tools/nni_cmd/config_utils.py

@@ -79,7 +79,7 @@ class Experiments:
         self.experiments[id]['port'] = port
         self.experiments[id]['startTime'] = time
         self.experiments[id]['endTime'] = 'N/A'
-        self.experiments[id]['status'] = 'running'
+        self.experiments[id]['status'] = 'INITIALIZED'
         self.experiments[id]['fileName'] = file_name
         self.experiments[id]['platform'] = platform
         self.write_file()

tools/nni_cmd/constants.py

@@ -30,6 +30,8 @@ WARNING_INFO = 'WARNING: %s'
 
 DEFAULT_REST_PORT = 8080
 
+REST_TIME_OUT = 20
+
 EXPERIMENT_SUCCESS_INFO = '\033[1;32;32mSuccessfully started experiment!\n\033[0m' \
                           '-----------------------------------------------------------------------\n' \
                           'The experiment id is %s\n'\

tools/nni_cmd/launcher.py

@@ -139,7 +139,7 @@ def set_trial_config(experiment_config, port, config_file_name):
     '''set trial configuration'''
     request_data = dict()
     request_data['trial_config'] = experiment_config['trial']
-    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 20)
+    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT)
     if check_response(response):
         return True
     else:
@@ -159,7 +159,7 @@ def set_remote_config(experiment_config, port, config_file_name):
     #set machine_list
     request_data = dict()
     request_data['machine_list'] = experiment_config['machineList']
-    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), 20)
+    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT)
     err_message = ''
     if not response or not check_response(response):
         if response is not None:
@@ -180,7 +180,7 @@ def setNNIManagerIp(experiment_config, port, config_file_name):
         return True, None
     ip_config_dict = dict()
     ip_config_dict['nni_manager_ip'] = { 'nniManagerIp' : experiment_config['nniManagerIp'] }
-    response = rest_put(cluster_metadata_url(port), json.dumps(ip_config_dict), 20)
+    response = rest_put(cluster_metadata_url(port), json.dumps(ip_config_dict), REST_TIME_OUT)
     err_message = None
     if not response or not response.status_code == 200:
         if response is not None:
@@ -195,7 +195,7 @@ def set_pai_config(experiment_config, port, config_file_name):
     '''set pai configuration''' 
     pai_config_data = dict()
     pai_config_data['pai_config'] = experiment_config['paiConfig']
-    response = rest_put(cluster_metadata_url(port), json.dumps(pai_config_data), 20)
+    response = rest_put(cluster_metadata_url(port), json.dumps(pai_config_data), REST_TIME_OUT)
     err_message = None
     if not response or not response.status_code == 200:
         if response is not None:
@@ -214,7 +214,7 @@ def set_kubeflow_config(experiment_config, port, config_file_name):
     '''set kubeflow configuration''' 
     kubeflow_config_data = dict()
     kubeflow_config_data['kubeflow_config'] = experiment_config['kubeflowConfig']
-    response = rest_put(cluster_metadata_url(port), json.dumps(kubeflow_config_data), 20)
+    response = rest_put(cluster_metadata_url(port), json.dumps(kubeflow_config_data), REST_TIME_OUT)
     err_message = None
     if not response or not response.status_code == 200:
         if response is not None:
@@ -233,7 +233,7 @@ def set_frameworkcontroller_config(experiment_config, port, config_file_name):
     '''set kubeflow configuration''' 
     frameworkcontroller_config_data = dict()
     frameworkcontroller_config_data['frameworkcontroller_config'] = experiment_config['frameworkcontrollerConfig']
-    response = rest_put(cluster_metadata_url(port), json.dumps(frameworkcontroller_config_data), 20)
+    response = rest_put(cluster_metadata_url(port), json.dumps(frameworkcontroller_config_data), REST_TIME_OUT)
     err_message = None
     if not response or not response.status_code == 200:
         if response is not None:
@@ -304,7 +304,7 @@ def set_experiment(experiment_config, mode, port, config_file_name):
         request_data['clusterMetaData'].append(
             {'key': 'trial_config', 'value': experiment_config['trial']})
 
-    response = rest_post(experiment_url(port), json.dumps(request_data), 20)
+    response = rest_post(experiment_url(port), json.dumps(request_data), REST_TIME_OUT)
     if check_response(response):
         return response
     else:
@@ -488,7 +488,7 @@ def resume_experiment(args):
         if experiment_dict.get(args.id) is None:
             print_error('Id %s not exist!' % args.id)
             exit(1)
-        if experiment_dict[args.id]['status'] == 'running':
+        if experiment_dict[args.id]['status'] != 'STOPPED':
             print_error('Experiment %s is running!' % args.id)
             exit(1)
         experiment_id = args.id

tools/nni_cmd/nnictl_utils.py

@@ -28,10 +28,25 @@ from .rest_utils import rest_get, rest_delete, check_rest_server_quick, check_re
 from .config_utils import Config, Experiments
 from .url_utils import trial_jobs_url, experiment_url, trial_job_id_url
 from .constants import NNICTL_HOME_DIR, EXPERIMENT_INFORMATION_FORMAT, EXPERIMENT_DETAIL_FORMAT, \
-     EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL
+     EXPERIMENT_MONITOR_INFO, TRIAL_MONITOR_HEAD, TRIAL_MONITOR_CONTENT, TRIAL_MONITOR_TAIL, REST_TIME_OUT
 from .common_utils import print_normal, print_error, print_warning, detect_process
 
-def update_experiment_status():
+def get_experiment_time(port):
+    '''get the startTime and endTime of an experiment'''
+    response = rest_get(experiment_url(port), REST_TIME_OUT)
+    if response and check_response(response):
+        content = convert_time_stamp_to_date(json.loads(response.text))
+        return content.get('startTime'), content.get('endTime')
+    return None, None
+
+def get_experiment_status(port):
+    '''get the status of an experiment'''
+    result, response = check_rest_server_quick(port)
+    if result:
+        return json.loads(response.text).get('status')
+    return None
+
+def update_experiment():
     '''Update the experiment status in config file'''
     experiment_config = Experiments()
     experiment_dict = experiment_config.get_all_experiments()
@@ -39,16 +54,26 @@ def update_experiment_status():
         return None
     for key in experiment_dict.keys():
         if isinstance(experiment_dict[key], dict):
-            if experiment_dict[key].get('status') == 'running':
+            if experiment_dict[key].get('status') != 'STOPPED':
                 nni_config = Config(experiment_dict[key]['fileName'])
                 rest_pid = nni_config.get_config('restServerPid')
                 if not detect_process(rest_pid):
-                    experiment_config.update_experiment(key, 'status', 'stopped')
+                    experiment_config.update_experiment(key, 'status', 'STOPPED')
+                    continue
+                rest_port = nni_config.get_config('restServerPort')
+                startTime, endTime = get_experiment_time(rest_port)
+                if startTime:
+                    experiment_config.update_experiment(key, 'startTime', startTime)
+                if endTime:
+                    experiment_config.update_experiment(key, 'endTime', endTime)
+                status = get_experiment_status(rest_port)
+                if status:
+                    experiment_config.update_experiment(key, 'status', status)
 
 def check_experiment_id(args):
     '''check if the id is valid
     '''
-    update_experiment_status()
+    update_experiment()
     experiment_config = Experiments()
     experiment_dict = experiment_config.get_all_experiments()
     if not experiment_dict:
@@ -58,13 +83,13 @@ def check_experiment_id(args):
         running_experiment_list = []
         for key in experiment_dict.keys():
             if isinstance(experiment_dict[key], dict):
-                if experiment_dict[key].get('status') == 'running':
+                if experiment_dict[key].get('status') != 'STOPPED':
                     running_experiment_list.append(key)
             elif isinstance(experiment_dict[key], list):
                 # if the config file is old version, remove the configuration from file
                 experiment_config.remove_experiment(key)
         if len(running_experiment_list) > 1:
-            print_error('There are multiple experiments running, please set the experiment id...')
+            print_error('There are multiple experiments, please set the experiment id...')
             experiment_information = ""
             for key in running_experiment_list:
                 experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
@@ -94,7 +119,7 @@ def parse_ids(args):
     5.If the id does not exist but match the prefix of an experiment id, nnictl will return the matched id
     6.If the id does not exist but match multiple prefix of the experiment ids, nnictl will give id information
     '''
-    update_experiment_status()
+    update_experiment()
     experiment_config = Experiments()
     experiment_dict = experiment_config.get_all_experiments()
     if not experiment_dict:
@@ -104,14 +129,14 @@ def parse_ids(args):
     running_experiment_list = []
     for key in experiment_dict.keys():
         if isinstance(experiment_dict[key], dict):
-            if experiment_dict[key].get('status') == 'running':
+            if experiment_dict[key].get('status') != 'STOPPED':
                 running_experiment_list.append(key)
         elif isinstance(experiment_dict[key], list):
             # if the config file is old version, remove the configuration from file
             experiment_config.remove_experiment(key)
     if not args.id:
         if len(running_experiment_list) > 1:
-            print_error('There are multiple experiments running, please set the experiment id...')
+            print_error('There are multiple experiments, please set the experiment id...')
             experiment_information = ""
             for key in running_experiment_list:
                 experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], \
@@ -207,7 +232,7 @@ def stop_experiment(args):
                             print_error(exception)
                     nni_config.set_config('tensorboardPidList', [])
             print_normal('Stop experiment success!')
-            experiment_config.update_experiment(experiment_id, 'status', 'stopped')
+            experiment_config.update_experiment(experiment_id, 'status', 'STOPPED')
             time_now = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))
             experiment_config.update_experiment(experiment_id, 'endTime', str(time_now))
 
@@ -221,7 +246,7 @@ def trial_ls(args):
         return
     running, response = check_rest_server_quick(rest_port)
     if running:
-        response = rest_get(trial_jobs_url(rest_port), 20)
+        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
         if response and check_response(response):
             content = json.loads(response.text)
             for index, value in enumerate(content):
@@ -242,7 +267,7 @@ def trial_kill(args):
         return
     running, _ = check_rest_server_quick(rest_port)
     if running:
-        response = rest_delete(trial_job_id_url(rest_port, args.id), 20)
+        response = rest_delete(trial_job_id_url(rest_port, args.id), REST_TIME_OUT)
         if response and check_response(response):
             print(response.text)
         else:
@@ -260,7 +285,7 @@ def list_experiment(args):
         return
     running, _ = check_rest_server_quick(rest_port)
     if running:
-        response = rest_get(experiment_url(rest_port), 20)
+        response = rest_get(experiment_url(rest_port), REST_TIME_OUT)
         if response and check_response(response):
             content = convert_time_stamp_to_date(json.loads(response.text))
             print(json.dumps(content, indent=4, sort_keys=True, separators=(',', ':')))
@@ -322,7 +347,7 @@ def log_trial(args):
         return
     running, response = check_rest_server_quick(rest_port)
     if running:
-        response = rest_get(trial_jobs_url(rest_port), 20)
+        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
         if response and check_response(response):
             content = json.loads(response.text)
             for trial in content:
@@ -362,18 +387,20 @@ def experiment_list(args):
     if not experiment_dict:
         print('There is no experiment running...')
         exit(1)
+    update_experiment()
     experiment_id_list = []
     if args.all and args.all == 'all':
         for key in experiment_dict.keys():
             experiment_id_list.append(key)
     else:
         for key in experiment_dict.keys():
-            if experiment_dict[key]['status'] == 'running':
+            if experiment_dict[key]['status'] != 'STOPPED':
                 experiment_id_list.append(key)
         if not experiment_id_list:
             print_warning('There is no experiment running...\nYou can use \'nnictl experiment list all\' to list all stopped experiments!')
     experiment_information = ""
     for key in experiment_id_list:
+        
         experiment_information += (EXPERIMENT_DETAIL_FORMAT % (key, experiment_dict[key]['status'], experiment_dict[key]['port'],\
         experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], experiment_dict[key]['endTime']))
     print(EXPERIMENT_INFORMATION_FORMAT % experiment_information)
@@ -382,8 +409,8 @@ def get_time_interval(time1, time2):
     '''get the interval of two times'''
     try:
         #convert time to timestamp
-        time1 = time.mktime(time.strptime(time1, '%Y-%m-%d %H:%M:%S'))
-        time2 = time.mktime(time.strptime(time2, '%Y-%m-%d %H:%M:%S'))
+        time1 = time.mktime(time.strptime(time1, '%Y/%m/%d %H:%M:%S'))
+        time2 = time.mktime(time.strptime(time2, '%Y/%m/%d %H:%M:%S'))
         seconds = (datetime.datetime.fromtimestamp(time2) - datetime.datetime.fromtimestamp(time1)).seconds
         #convert seconds to day:hour:minute:second
         days = seconds / 86400
@@ -403,21 +430,21 @@ def show_experiment_info():
     if not experiment_dict:
         print('There is no experiment running...')
         exit(1)
+    update_experiment()
     experiment_id_list = []
     for key in experiment_dict.keys():
-        if experiment_dict[key]['status'] == 'running':
+        if experiment_dict[key]['status'] != 'STOPPED':
             experiment_id_list.append(key)
     if not experiment_id_list:
         print_warning('There is no experiment running...')
         return
     for key in experiment_id_list:
-        current_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
         print(EXPERIMENT_MONITOR_INFO % (key, experiment_dict[key]['status'], experiment_dict[key]['port'], \
-             experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], get_time_interval(experiment_dict[key]['startTime'], current_time)))
+             experiment_dict[key].get('platform'), experiment_dict[key]['startTime'], get_time_interval(experiment_dict[key]['startTime'], experiment_dict[key]['endTime'])))
         print(TRIAL_MONITOR_HEAD)
         running, response = check_rest_server_quick(experiment_dict[key]['port'])
         if running:
-            response = rest_get(trial_jobs_url(experiment_dict[key]['port']), 20)
+            response = rest_get(trial_jobs_url(experiment_dict[key]['port']), REST_TIME_OUT)
             if response and check_response(response):
                 content = json.loads(response.text)
                 for index, value in enumerate(content):
@@ -433,7 +460,7 @@ def monitor_experiment(args):
     while True:
         try:
             os.system('clear')
-            update_experiment_status()
+            update_experiment()
             show_experiment_info()
             time.sleep(args.time)
         except KeyboardInterrupt:

tools/nni_cmd/rest_utils.py

@@ -22,6 +22,7 @@
 import time
 import requests
 from .url_utils import check_status_url
+from .constants import REST_TIME_OUT
 
 def rest_put(url, data, timeout):
     '''Call rest put method'''
@@ -61,7 +62,7 @@ def check_rest_server(rest_port):
     '''Check if restful server is ready'''
     retry_count = 5
     for _ in range(retry_count):
-        response = rest_get(check_status_url(rest_port), 20)
+        response = rest_get(check_status_url(rest_port), REST_TIME_OUT)
         if response:
             if response.status_code == 200:
                 return True, response

tools/nni_cmd/tensorboard_utils.py

@@ -144,7 +144,7 @@ def start_tensorboard(args):
     running, response = check_rest_server_quick(rest_port)
     trial_content = None
     if running:
-        response = rest_get(trial_jobs_url(rest_port), 20)
+        response = rest_get(trial_jobs_url(rest_port), REST_TIME_OUT)
         if response and check_response(response):
             trial_content = json.loads(response.text)
         else:

tools/nni_cmd/updater.py

@@ -27,6 +27,7 @@ from .config_utils import Config
 from .common_utils import get_json_content
 from .nnictl_utils import check_experiment_id, get_experiment_port, get_config_filename
 from .launcher_utils import parse_time
+from .constants import REST_TIME_OUT
 
 def validate_digit(value, start, end):
     '''validate if a digit is valid'''
@@ -62,11 +63,11 @@ def update_experiment_profile(args, key, value):
     rest_port = nni_config.get_config('restServerPort')
     running, _ = check_rest_server_quick(rest_port)
     if running:
-        response = rest_get(experiment_url(rest_port), 20)
+        response = rest_get(experiment_url(rest_port), REST_TIME_OUT)
         if response and check_response(response):
             experiment_profile = json.loads(response.text)
             experiment_profile['params'][key] = value
-            response = rest_put(experiment_url(rest_port)+get_query_type(key), json.dumps(experiment_profile), 20)
+            response = rest_put(experiment_url(rest_port)+get_query_type(key), json.dumps(experiment_profile), REST_TIME_OUT)
             if response and check_response(response):
                 return response
     else:

tools/nni_trial_tool/constants.py

@@ -35,6 +35,7 @@ STDOUT_FULL_PATH = os.path.join(LOG_DIR, 'stdout')
 STDERR_FULL_PATH = os.path.join(LOG_DIR, 'stderr')
 
 STDOUT_API = '/stdout'
+VERSION_API = '/version'
 NNI_SYS_DIR = os.environ['NNI_SYS_DIR']
 NNI_TRIAL_JOB_ID = os.environ['NNI_TRIAL_JOB_ID']
 NNI_EXP_ID = os.environ['NNI_EXP_ID']
\ No newline at end of file

tools/nni_trial_tool/trial_keeper.py

@@ -27,14 +27,18 @@ import shlex
 import re
 import sys
 import select
+import json
 from pyhdfs import HdfsClient
 import pkg_resources
+from .rest_utils import rest_post
+from .url_utils import gen_send_stdout_url, gen_send_version_url
 
 from .constants import HOME_DIR, LOG_DIR, NNI_PLATFORM, STDOUT_FULL_PATH, STDERR_FULL_PATH
 from .hdfsClientUtility import copyDirectoryToHdfs, copyHdfsDirectoryToLocal
 from .log_utils import LogType, nni_log, RemoteLogger, PipeLogReader, StdOutputType
 
 logger = logging.getLogger('trial_keeper')
+regular = re.compile('v?(?P<version>[0-9](\.[0-9]){0,1}).*')
 
 def main_loop(args):
     '''main loop logic for trial keeper'''
@@ -110,21 +114,27 @@ def check_version(args):
         #package nni does not exist, try nni-tool package
         nni_log(LogType.Error, 'Package nni does not exist!')
         os._exit(1)
-    if not args.version:
+    if not args.nni_manager_version:
         # skip version check
         nni_log(LogType.Warning, 'Skipping version check!')
     else:
-        regular = re.compile('v?(?P<version>[0-9](\.[0-9]){0,2}).*')
         try:
             trial_keeper_version = regular.search(trial_keeper_version).group('version')
             nni_log(LogType.Info, 'trial_keeper_version is {0}'.format(trial_keeper_version))
-            training_service_version = regular.search(args.version).group('version')
-            nni_log(LogType.Info, 'training_service_version is {0}'.format(training_service_version))
-            if trial_keeper_version != training_service_version:
+            nni_manager_version = regular.search(args.nni_manager_version).group('version')
+            nni_log(LogType.Info, 'nni_manager_version is {0}'.format(nni_manager_version))
+            log_entry = {}
+            if trial_keeper_version != nni_manager_version:
                 nni_log(LogType.Error, 'Version does not match!')
+                error_message = 'NNIManager version is {0}, TrialKeeper version is {1}, NNI version does not match!'.format(nni_manager_version, trial_keeper_version)
+                log_entry['tag'] = 'VCFail'
+                log_entry['msg'] = error_message
+                rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, False)
                 os._exit(1)
             else:
                 nni_log(LogType.Info, 'Version match!')
+                log_entry['tag'] = 'VCSuccess'
+                rest_post(gen_send_version_url(args.nnimanager_ip, args.nnimanager_port), json.dumps(log_entry), 10, False)
         except AttributeError as err:
             nni_log(LogType.Error, err)
 
@@ -142,7 +152,7 @@ if __name__ == '__main__':
     PARSER.add_argument('--pai_user_name', type=str, help='the username of hdfs')
     PARSER.add_argument('--nni_hdfs_exp_dir', type=str, help='nni experiment directory in hdfs')
     PARSER.add_argument('--webhdfs_path', type=str, help='the webhdfs path used in webhdfs URL')
-    PARSER.add_argument('--version', type=str, help='the nni version transmitted from trainingService')
+    PARSER.add_argument('--nni_manager_version', type=str, help='the nni version transmitted from nniManager')
     PARSER.add_argument('--log_collection', type=str, help='set the way to collect log in trialkeeper')
     args, unknown = PARSER.parse_known_args()
     if args.trial_command is None:

tools/nni_trial_tool/url_utils.py

@@ -18,8 +18,12 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-from .constants import API_ROOT_URL, BASE_URL, STDOUT_API, NNI_TRIAL_JOB_ID, NNI_EXP_ID
+from .constants import API_ROOT_URL, BASE_URL, STDOUT_API, NNI_TRIAL_JOB_ID, NNI_EXP_ID, VERSION_API
 
 def gen_send_stdout_url(ip, port):
     '''Generate send stdout url'''
     return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, STDOUT_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID)
+
+def gen_send_version_url(ip, port):
+    '''Generate send error url'''
+    return '{0}:{1}{2}{3}/{4}/{5}'.format(BASE_URL.format(ip), port, API_ROOT_URL, VERSION_API, NNI_EXP_ID, NNI_TRIAL_JOB_ID)
\ No newline at end of file