NNI dogfood version 1

src/sdk/pynni/nni/__init__.py

+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+# associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute,
+# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# ==================================================================================================
+
+
+# pylint: disable=wildcard-import
+
+from .trial import *
+from .smartparam import *

src/sdk/pynni/nni/assessor.py

+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+# associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute,
+# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# ==================================================================================================
+
+
+from collections import defaultdict
+from enum import Enum
+import logging
+import os
+
+import json_tricks
+
+from .common import init_logger
+from .protocol import CommandType, send, receive
+
+
+init_logger('assessor.log')
+_logger = logging.getLogger(__name__)
+
+
+class AssessResult(Enum):
+    Good = True
+    Bad = False
+
+
+class Assessor:
+    # pylint: disable=no-self-use,unused-argument
+
+    def assess_trial(self, trial_job_id, trial_history):
+        """Determines whether a trial should be killed. Must override.
+        trial_job_id: identifier of the trial (str).
+        trial_history: a list of intermediate result objects.
+        Returns AssessResult.Good or AssessResult.Bad.
+        """
+        raise NotImplementedError('Assessor: assess_trial not implemented')
+
+    def trial_end(self, trial_job_id, success):
+        """Invoked when a trial is completed or terminated. Do nothing by default.
+        trial_job_id: identifier of the trial (str).
+        success: True if the trial successfully completed; False if failed or terminated.
+        """
+        pass
+
+    def load_checkpoint(self, path):
+        """Load the checkpoint of assessor.
+        path: checkpoint directory of assessor
+        """
+        _logger.info('Load checkpoint ignored by assessor')
+
+    def save_checkpoint(self, path):
+        """Save the checkpoint of assessor.
+        path: checkpoint directory of assessor
+        """
+        _logger.info('Save checkpoint ignored by assessor')
+
+    def request_save_checkpoint(self):
+        """Request to save the checkpoint of assessor
+        """
+        self.save_checkpoint(os.getenv('NNI_CHECKPOINT_DIRECTORY'))
+
+    def run(self):
+        """Run the assessor.
+        This function will never return unless raise.
+        """
+        mode = os.getenv('NNI_MODE')
+        if mode == 'resume':
+            self.load_checkpoint(os.getenv('NNI_CHECKPOINT_DIRECTORY'))
+        while _handle_request(self):
+            pass
+        _logger.info('Terminated by NNI manager')
+
+
+_trial_history = defaultdict(dict)
+'''key: trial job ID; value: intermediate results, mapping from sequence number to data'''
+
+_ended_trials = set()
+'''trial_job_id of all ended trials.
+We need this because NNI manager may send metrics after reporting a trial ended.
+TODO: move this logic to NNI manager
+'''
+
+def _sort_history(history):
+    ret = [ ]
+    for i, _ in enumerate(history):
+        if i in history:
+            ret.append(history[i])
+        else:
+            break
+    return ret
+
+def _handle_request(assessor):
+    _logger.debug('waiting receive_message')
+
+    command, data = receive()
+
+    _logger.debug(command)
+    _logger.debug(data)
+
+    if command is CommandType.Terminate:
+        return False
+
+    data = json_tricks.loads(data)
+
+    if command is CommandType.ReportMetricData:
+        if data['type'] != 'PERIODICAL':
+            return True
+
+        trial_job_id = data['trial_job_id']
+        if trial_job_id in _ended_trials:
+            return True
+
+        history = _trial_history[trial_job_id]
+        history[data['sequence']] = data['value']
+        ordered_history = _sort_history(history)
+        if len(ordered_history) < data['sequence']:  # no user-visible update since last time
+            return True
+
+        result = assessor.assess_trial(trial_job_id, ordered_history)
+        if isinstance(result, bool):
+            result = AssessResult.Good if result else AssessResult.Bad
+        elif not isinstance(result, AssessResult):
+            msg = 'Result of Assessor.assess_trial must be an object of AssessResult, not %s'
+            raise RuntimeError(msg % type(result))
+
+        if result is AssessResult.Bad:
+            _logger.debug('BAD, kill %s', trial_job_id)
+            send(CommandType.KillTrialJob, json_tricks.dumps(trial_job_id))
+        else:
+            _logger.debug('GOOD')
+
+    elif command is CommandType.TrialEnd:
+        trial_job_id = data['trial_job_id']
+        _ended_trials.add(trial_job_id)
+        if trial_job_id in _trial_history:
+            _trial_history.pop(trial_job_id)
+            assessor.trial_end(trial_job_id, data['event'] == 'SUCCEEDED')
+
+    else:
+        raise AssertionError('Unsupported command: %s' % command)
+
+    return True

src/sdk/pynni/nni/common.py

+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+# associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute,
+# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# ==================================================================================================
+
+
+from collections import namedtuple
+from io import TextIOBase
+import logging
+import os
+import sys
+
+
+def _load_env_args():
+    args = {
+        'platform': os.environ.get('NNI_PLATFORM'),
+        'trial_job_id': os.environ.get('NNI_TRIAL_JOB_ID'),
+        'log_dir': os.environ.get('NNI_LOG_DIRECTORY'),
+        'role': os.environ.get('NNI_ROLE'),
+    }
+    return namedtuple('EnvArgs', args.keys())(**args)
+
+env_args = _load_env_args()
+'''Arguments passed from environment'''
+
+
+class _LoggerFile(TextIOBase):
+    def __init__(self, logger):
+        self.logger = logger
+
+    def write(self, s):
+        if s != '\n':  # ignore line break, since logger will add it
+            self.logger.info(s)
+        return len(s)
+
+
+def init_logger(logger_file_path):
+    """Initialize root logger.
+    This will redirect anything from logging.getLogger() as well as stdout to specified file.
+    logger_file_path: path of logger file (path-like object).
+    """
+    if env_args.platform == 'unittest':
+        logger_file_path = 'unittest.log'
+    elif env_args.log_dir is not None:
+        logger_file_path = os.path.join(env_args.log_dir, logger_file_path)
+
+    fmt = '[%(asctime)s] %(levelname)s (%(name)s) %(message)s'
+    datefmt = '%Y-%m-%d %H:%M:%S'
+    formatter = logging.Formatter(fmt, datefmt)
+
+    handler = logging.FileHandler(logger_file_path)
+    handler.setFormatter(formatter)
+
+    root_logger = logging.getLogger()
+    root_logger.addHandler(handler)
+    root_logger.setLevel(logging.DEBUG)
+
+    # these modules are too verbose
+    logging.getLogger('matplotlib').setLevel(logging.INFO)
+
+    sys.stdout = _LoggerFile(logging.getLogger('print'))

src/sdk/pynni/nni/darkopt_assessor/__main__.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import argparse
+import logging
+
+from .darkopt_assessor import DarkoptAssessor, OptimizeMode
+
+
+logger = logging.getLogger('nni.contribution.darkopt_assessor')
+logger.debug('START')
+
+
+def _main():
+    # run accessor for mnist:
+    # python -m nni.contribution.darkopt_assessor --best_score=0.90 --period=200 --threshold=0.9 --optimize_mode=maximize
+    parser = argparse.ArgumentParser(
+        description='parse command line parameters.')
+    parser.add_argument('--best_score', type=float,
+                        help='Expected best score for Assessor.')
+    parser.add_argument('--period', type=int,
+                        help='Expected period for Assessor.')
+    parser.add_argument('--threshold', type=float,
+                        help='Threshold for Assessor.')
+    parser.add_argument('--optimize_mode', type=str, default='maximize',
+            help='Select optimize mode for Assessor: minimize or maximize.')
+    FLAGS, unparsed = parser.parse_known_args()
+
+    if FLAGS.optimize_mode not in [ mode.value for mode in OptimizeMode ]:
+        raise AttributeError('Unsupported optimzie mode "%s"' % FLAGS.optimize_mode)
+
+    logger.debug('params:' + str(FLAGS))
+
+    assessor = DarkoptAssessor(FLAGS.best_score, FLAGS.period, FLAGS.threshold, OptimizeMode(FLAGS.optimize_mode))
+    assessor.run()
+
+
+try:
+    _main()
+except Exception as exception:
+    logger.exception(exception)
\ No newline at end of file

src/sdk/pynni/nni/darkopt_assessor/darkopt2.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import numpy as np
+import pymc3
+import scipy.stats
+
+class EnsembleSamplingPredictor(object):
+    def __init__(self):
+        self.traces = None
+
+    def fit(self, x, y):
+        x = np.asarray(x)
+        y = np.asarray(y)
+        self.traces = sample_ensemble(x, y)
+
+    def predict_proba_less_than(self, x, y):
+        return np.mean([
+            predict_proba_less_than_ensemble(x, y, trace)
+            for trace in self.traces
+        ], axis=0)
+
+
+curves = [
+    ('vapore_pressure', 3, lambda x, p: p[0] * np.exp(p[1] / (1 + x) + p[2] * np.log1p(x))),
+    ('weibull', 3, lambda x, p: p[0] - p[1] * np.exp(-p[2] * x)),
+]
+
+
+def _single(x, y, curve):
+    name, n_params, func = curve
+    with pymc3.Model() as model_single:
+        params = pymc3.Flat(name, shape=n_params)
+        mu = func(x, params)
+        sd = pymc3.Uniform('sd', lower=1e-9, upper=1e-1)
+        pymc3.Normal('y_obs', mu=mu, sd=sd, observed=y)
+        map_estimate = pymc3.find_MAP()
+        return map_estimate[name]
+
+
+def sample_ensemble(x, y):
+    start = { curve[0]: _single(x, y, curve) for curve in curves }
+    start['weights_unnormalized_interval_'] = np.zeros(len(curves))
+    start['sd_interval_'] = 0
+    with pymc3.Model() as model_ensemble:
+        mu_single = []
+        for name, n_params, func in curves:
+            params = pymc3.Flat(name, shape=n_params)
+            mu_single.append(func(x, params))
+        weights_unnormalized = pymc3.Uniform(
+            'weights_unnnormalized', lower=0, upper=1, shape=len(curves))
+        weights_normalized = pymc3.Deterministic(
+            'weights_normalized', weights_unnormalized / weights_unnormalized.sum())
+        mu_ensemble = weights_normalized.dot(mu_single)
+        sd1 = pymc3.Uniform('sd', lower=1e-9, upper=1e-1)
+        pymc3.Deterministic('sd1', sd1)
+        pymc3.Normal('y_obs', mu=mu_ensemble, observed=y, sd=sd1)
+        return pymc3.sample(start=start, step=pymc3.Metropolis(), draws=1000)
+
+
+def predict_proba_less_than_ensemble(x, y, param):
+    ps = [func(x, param[name]) for name, _, func in curves]
+    mu = param['weights_normalized'].dot(ps)
+    sd1 = param['sd1']
+    return scipy.stats.norm.cdf(y, loc=mu, scale=sd1)
\ No newline at end of file

src/sdk/pynni/nni/darkopt_assessor/darkopt_assessor.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+from nni.assessor import Assessor, AssessResult
+
+from .darkopt2 import EnsembleSamplingPredictor
+
+from enum import Enum
+import logging
+
+logger = logging.getLogger('nni.contribution.darkopt_assessor')
+
+
+class OptimizeMode(Enum):
+    Maximize = 'maximize'
+    Minimize = 'minimize'
+
+
+class DarkoptAssessor(Assessor):
+    def __init__(self, best_score, period, threshold, optimize_mode):
+        self.best_score = best_score
+        self.period = period
+        self.threshold = threshold
+        self.optimize_mode = optimize_mode
+
+        self.predictor = EnsembleSamplingPredictor()
+        if self.optimize_mode is OptimizeMode.Minimize:
+            self.best_score = -self.best_score
+
+
+    def assess_trial(self, trial_job_id, history):
+        '''
+        assess_trial
+        '''
+        logger.debug('assess_trial %s' % history)
+        if self.optimize_mode is OptimizeMode.Minimize:
+            history = [ -x for x in history ]
+
+        max_ = max(history)
+        if max_ > self.best_score:
+            self.best_score = max_
+            return AssessResult.Good
+
+        self.predictor.fit(list(range(len(history))), history)
+        proba_worse = self.predictor.predict_proba_less_than(self.period, self.best_score)
+
+        if proba_worse > self.threshold:
+            return AssessResult.Bad
+        else:
+            return AssessResult.Good
\ No newline at end of file

src/sdk/pynni/nni/darkopt_assessor/test.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import argparse
+import logging
+import random
+from enum import Enum
+
+from .darkopt_assessor import DarkoptAssessor
+from nni.assessor import AssessResult
+
+
+logger = logging.getLogger('nni.contrib.darkopt_assessor')
+logger.debug('START')
+
+class OptimizeMode(Enum):
+    Maximize = 'maximize'
+    Minimize = 'minimize'
+
+def test():
+    '''
+    tests.
+    '''
+    # run accessor for mnist:
+    # python -m nni.contribution.darkopt_assessor --best_score=0.90 --period=200 --threshold=0.9 --optimize_mode=maximize
+    parser = argparse.ArgumentParser(
+        description='parse command line parameters.')
+    parser.add_argument('--best_score', type=float,
+                        help='Expected best score for Assessor.')
+    parser.add_argument('--period', type=int,
+                        help='Expected period for Assessor.')
+    parser.add_argument('--threshold', type=float,
+                        help='Threshold for Assessor.')
+    parser.add_argument('--optimize_mode', type=str, default='maximize',
+            help='Select optimize mode for Assessor: minimize or maximize.')
+    FLAGS, unparsed = parser.parse_known_args()
+
+    if FLAGS.optimize_mode not in [ mode.value for mode in OptimizeMode ]:
+        raise AttributeError('Unsupported optimzie mode "%s"' % FLAGS.optimize_mode)
+
+    lcs = [[0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1],
+           [0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2],
+           [0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3],
+           [0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]]
+    #lcs = [[1,1,1,1,1,1,1,1,1,1],
+    #       [1,1,1,1,1,1,1,1,1,1],
+    #       [1,1,1,1,1,1,1,1,1,1]]
+
+    assessor = DarkoptAssessor(FLAGS.best_score, FLAGS.period, FLAGS.threshold, FLAGS.optimize_mode)
+    for i in range(4):
+        #lc = []
+        for k in range(10):
+            #d = random.randint(i*100+0, i*100+100)
+            #lc.append(d)
+            ret = assessor.assess_trial(i, lcs[i][:k+1])
+            print('result: %d', ret)
+
+try:
+    test()
+except Exception as exception:
+    logger.exception(exception)
+    raise
\ No newline at end of file

src/sdk/pynni/nni/evolution_tuner/__main__.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+__main__.py contains a main function to call Evolution.
+'''
+
+import argparse
+import logging
+
+from .evolution_tuner import EvolutionTuner, OptimizeMode
+
+
+logger = logging.getLogger('nni.examples.evolution_tuner')
+logger.debug('START')
+
+
+def main():
+    '''
+    main function.
+    '''
+    parser = argparse.ArgumentParser(description='parse command line parameters.')
+    parser.add_argument('--optimize_mode', type=str, default='maximize',
+                        help='Select optimize mode for Tuner: minimize or maximize.')
+    FLAGS, _ = parser.parse_known_args()
+
+    if FLAGS.optimize_mode not in [mode.value for mode in OptimizeMode]:
+        raise AttributeError('Unsupported optimize mode "%s"' % FLAGS.optimize_mode)
+
+    tuner = EvolutionTuner(FLAGS.optimize_mode)
+    tuner.run()
+
+
+try:
+    main()
+except Exception as exception:
+    logger.exception(exception)
+    raise

src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+evolution_tuner.py including:
+    class OptimizeMode
+    class Individual
+    class EvolutionTuner
+if OptimizeMode is 'minimize', it means the tuner need to minimize the reward
+that received from Trial.
+if OptimizeMode is 'maximize', it means the tuner need to maximize the reward
+that received from Trial.
+'''
+
+import copy
+from enum import Enum, unique
+import random
+
+import numpy as np
+
+from nni.tuner import Tuner
+from . import parameter_expressions
+
+
+@unique
+class OptimizeMode(Enum):
+    '''
+    Optimize Mode class
+    '''
+    Minimize = 'minimize'
+    Maximize = 'maximize'
+
+
+@unique
+class NodeType(Enum):
+    '''
+    Node Type class
+    '''
+    Root = 'root'
+    Type = '_type'
+    Value = '_value'
+    Index = '_index'
+
+
+def json2space(x, oldy=None, name=NodeType.Root.value):
+    '''
+    Change search space from json format to hyperopt format
+    '''
+    y = list()
+    if isinstance(x, dict):
+        if NodeType.Type.value in x.keys():
+            _type = x[NodeType.Type.value]
+            name = name + '-' + _type
+            if _type == 'choice':
+                if oldy != None:
+                    _index = oldy[NodeType.Index.value]
+                    y += json2space(x[NodeType.Value.value][_index],
+                                    oldy[NodeType.Value.value], name=name+'[%d]' % _index)
+                else:
+                    y += json2space(x[NodeType.Value.value], None, name=name)
+            y.append(name)
+        else:
+            for key in x.keys():
+                y += json2space(x[key], (oldy[key] if oldy !=
+                                         None else None), name+"[%s]" % str(key))
+    elif isinstance(x, list):
+        for i, x_i in enumerate(x):
+            y += json2space(x_i, (oldy[i] if oldy !=
+                                  None else None), name+"[%d]" % i)
+    else:
+        pass
+    return y
+
+
+def json2paramater(x, is_rand, random_state, oldy=None, Rand=False, name=NodeType.Root.value):
+    '''
+    Json to pramaters.
+    '''
+    if isinstance(x, dict):
+        if NodeType.Type.value in x.keys():
+            _type = x[NodeType.Type.value]
+            _value = x[NodeType.Value.value]
+            name = name + '-' + _type
+            Rand |= is_rand[name]
+            if Rand is True:
+                if _type == 'choice':
+                    _index = random_state.randint(len(_value))
+                    y = {
+                        NodeType.Index.value: _index,
+                        NodeType.Value.value: json2paramater(x[NodeType.Value.value][_index],
+                                                             is_rand,
+                                                             random_state,
+                                                             None,
+                                                             Rand,
+                                                             name=name+"[%d]" % _index)
+                    }
+                else:
+                    y = eval('parameter_expressions.' +
+                             _type)(*(_value + [random_state]))
+            else:
+                y = copy.deepcopy(oldy)
+        else:
+            y = dict()
+            for key in x.keys():
+                y[key] = json2paramater(x[key], is_rand, random_state, oldy[key]
+                                        if oldy != None else None, Rand, name + "[%s]" % str(key))
+    elif isinstance(x, list):
+        y = list()
+        for i, x_i in enumerate(x):
+            y.append(json2paramater(x_i, is_rand, random_state, oldy[i]
+                                    if oldy != None else None, Rand, name + "[%d]" % i))
+    else:
+        y = copy.deepcopy(x)
+    return y
+
+
+def _split_index(params):
+    result = {}
+    for key in params:
+        if isinstance(params[key], dict):
+            value = params[key]['_value']
+        else:
+            value = params[key]
+        result[key] = value
+    return result
+
+
+class Individual(object):
+    '''
+    Indicidual class to store the indv info.
+    '''
+    def __init__(self, config=None, info=None, result=None, save_dir=None):
+        self.config = config
+        self.result = result
+        self.info = info
+        self.restore_dir = None
+        self.save_dir = save_dir
+
+    def __str__(self):
+        return "info: " + str(self.info) + \
+            ", config :" + str(self.config) + ", result: " + str(self.result)
+
+    def mutation(self, config=None, info=None, save_dir=None):
+        self.result = None
+        self.config = config
+        self.restore_dir = self.save_dir
+        self.save_dir = save_dir
+        self.info = info
+
+
+class EvolutionTuner(Tuner):
+    '''
+    EvolutionTuner is tuner using evolution algorithm.
+    '''
+
+    def __init__(self, optimize_mode, population_size=32):
+        self.optimize_mode = OptimizeMode(optimize_mode)
+        self.population_size = population_size
+        self.trial_result = []
+        self.searchspace_json = None
+        self.total_data = {}
+        self.random_state = None
+        self.population = None
+        self.space = None
+
+    def update_search_space(self, search_space):
+        '''
+        Update search space
+        search_space: search_space the json file that user pre-defined.
+        '''
+        self.searchspace_json = search_space
+        self.space = json2space(self.searchspace_json)
+
+        self.random_state = np.random.RandomState()
+        self.population = []
+        is_rand = dict()
+        for item in self.space:
+            is_rand[item] = True
+        for _ in range(self.population_size):
+            config = json2paramater(
+                self.searchspace_json, is_rand, self.random_state)
+            self.population.append(Individual(config=config))
+
+    def generate_parameters(self, parameter_id):
+        """Returns a set of trial (hyper-)parameters, as a serializable object.
+        parameter_id : int
+        """
+        if not self.population:
+            raise RuntimeError('The population is empty')
+        pos = -1
+        for i in range(len(self.population)):
+            if self.population[i].result is None:
+                pos = i
+                break
+        if pos != -1:
+            indiv = copy.deepcopy(self.population[pos])
+            self.population.pop(pos)
+            total_config = indiv.config
+        else:
+            random.shuffle(self.population)
+            if self.population[0].result < self.population[1].result:
+                self.population[0] = self.population[1]
+
+            # mutation
+            space = json2space(self.searchspace_json,
+                               self.population[0].config)
+            is_rand = dict()
+            mutation_pos = space[random.randint(0, len(space)-1)]
+            for i in range(len(self.space)):
+                is_rand[self.space[i]] = (self.space[i] == mutation_pos)
+            config = json2paramater(
+                self.searchspace_json, is_rand, self.random_state, self.population[0].config)
+            self.population.pop(1)
+            # remove "_index" from config and save params-id
+
+            total_config = config
+        self.total_data[parameter_id] = total_config
+        config = _split_index(total_config)
+        return config
+
+    def receive_trial_result(self, parameter_id, parameters, reward):
+        '''
+        Record an observation of the objective function
+        parameters: dict of parameters
+        reward: reward of one trial
+        '''
+        if parameter_id not in self.total_data:
+            raise RuntimeError('Received parameter_id not in total_data.')
+        # restore the paramsters contains "_index"
+        params = self.total_data[parameter_id]
+
+        if self.optimize_mode == OptimizeMode.Minimize:
+            reward = -reward
+
+        indiv = Individual(config=params, result=reward)
+        self.population.append(indiv)

src/sdk/pynni/nni/evolution_tuner/parameter_expressions.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+parameter_expression.py
+'''
+
+import numpy as np
+
+
+def choice(options, random_state):
+    '''
+    options: 1-D array-like or int
+    random_state: an object of numpy.random.RandomState
+    '''
+    return random_state.choice(options)
+
+
+def randint(upper, random_state):
+    '''
+    upper: an int that represent an upper bound
+    random_state: an object of numpy.random.RandomState
+    '''
+    return random_state.randint(upper)
+
+
+def uniform(low, high, random_state):
+    '''
+    low: an float that represent an lower bound
+    high: an float that represent an upper bound
+    random_state: an object of numpy.random.RandomState
+    '''
+    return random_state.uniform(low, high)
+
+
+def quniform(low, high, q, random_state):
+    '''
+    low: an float that represent an lower bound
+    high: an float that represent an upper bound
+    q: sample step
+    random_state: an object of numpy.random.RandomState
+    '''
+    return np.round(uniform(low, high, random_state) / q) * q
+
+
+def loguniform(low, high, random_state):
+    '''
+    low: an float that represent an lower bound
+    high: an float that represent an upper bound
+    random_state: an object of numpy.random.RandomState
+    '''
+    return np.exp(uniform(low, high, random_state))
+
+
+def qloguniform(low, high, q, random_state):
+    '''
+    low: an float that represent an lower bound
+    high: an float that represent an upper bound
+    q: sample step
+    random_state: an object of numpy.random.RandomState
+    '''
+    return np.round(loguniform(low, high, random_state) / q) * q
+
+
+def normal(mu, sigma, random_state):
+    '''
+    The probability density function of the normal distribution,
+    first derived by De Moivre and 200 years later by both Gauss and Laplace independently.
+    mu: float or array_like of floats
+        Mean (“centre”) of the distribution.
+    sigma: float or array_like of floats
+           Standard deviation (spread or “width”) of the distribution.
+    random_state: an object of numpy.random.RandomState
+    '''
+    return random_state.normal(mu, sigma)
+
+
+def qnormal(mu, sigma, q, random_state):
+    '''
+    mu: float or array_like of floats
+    sigma: float or array_like of floats
+    q: sample step
+    random_state: an object of numpy.random.RandomState
+    '''
+    return np.round(normal(mu, sigma, random_state) / q) * q
+
+
+def lognormal(mu, sigma, random_state):
+    '''
+    mu: float or array_like of floats
+    sigma: float or array_like of floats
+    random_state: an object of numpy.random.RandomState
+    '''
+    return np.exp(normal(mu, sigma, random_state))
+
+
+def qlognormal(mu, sigma, q, random_state):
+    '''
+    mu: float or array_like of floats
+    sigma: float or array_like of floats
+    q: sample step
+    random_state: an object of numpy.random.RandomState
+    '''
+    return np.round(lognormal(mu, sigma, random_state) / q) * q

src/sdk/pynni/nni/hyperopt_tuner/__main__.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+__main__.py
+'''
+
+import argparse
+import logging
+
+from .hyperopt_tuner import HyperoptTuner, OptimizeMode
+
+
+logger = logging.getLogger('nni.examples.hyperopt_tuner')
+logger.debug('START')
+
+
+def main():
+    '''
+    main function.
+    '''
+    parser = argparse.ArgumentParser(description='parse command line parameters.')
+    parser.add_argument('--optimize_mode', type=str, default='maximize',
+                        help='Select optimize mode for Tuner: minimize or maximize.')
+    parser.add_argument('--algorithm_name', type=str, default='tpe',
+                        help='Select algorithm for Tuner: tpe, random_search or anneal.')
+    FLAGS, _ = parser.parse_known_args()
+
+    if FLAGS.optimize_mode not in [mode.value for mode in OptimizeMode]:
+        raise AttributeError('Unsupported optimize mode "%s"' % FLAGS.optimize_mode)
+
+    tuner = HyperoptTuner(FLAGS.algorithm_name, FLAGS.optimize_mode)
+    tuner.run()
+
+
+try:
+    main()
+except Exception as exception:
+    logger.exception(exception)
+    raise

src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge,
+# to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+'''
+hyperopt_tuner.py
+'''
+
+import copy
+import logging
+
+from enum import Enum, unique
+import numpy as np
+
+import hyperopt as hp
+from nni.tuner import Tuner
+
+logger = logging.getLogger('hyperopt_AutoML')
+
+
+@unique
+class OptimizeMode(Enum):
+    '''
+    Oprimize Mode class
+    '''
+    Minimize = 'minimize'
+    Maximize = 'maximize'
+
+
+ROOT = 'root'
+TYPE = '_type'
+VALUE = '_value'
+INDEX = '_index'
+
+
+def json2space(in_x, name=ROOT):
+    '''
+    Change json to search space in hyperopt.
+	'''
+    out_y = copy.deepcopy(in_x)
+    if isinstance(in_x, dict):
+        if TYPE in in_x.keys():
+            _type = in_x[TYPE]
+            name = name + '-' + _type
+            _value = json2space(in_x[VALUE], name=name)
+            if _type == 'choice':
+                out_y = eval('hp.hp.'+_type)(name, _value)
+            else:
+                out_y = eval('hp.hp.' + _type)(name, *_value)
+        else:
+            out_y = dict()
+            for key in in_x.keys():
+                out_y[key] = json2space(in_x[key], name+"[%s]" % str(key))
+    elif isinstance(in_x, list):
+        out_y = list()
+        for i, x_i in enumerate(in_x):
+            out_y.append(json2space(x_i, name+"[%d]" % i))
+    else:
+        logger.info('in_x is not a dict or a list in json2space fuinction %s', str(in_x))
+    return out_y
+
+
+def json2paramater(in_x, paramater, name=ROOT):
+    '''
+    Change json to parameters.
+	'''
+    out_y = copy.deepcopy(in_x)
+    if isinstance(in_x, dict):
+        if TYPE in in_x.keys():
+            _type = in_x[TYPE]
+            name = name + '-' + _type
+            if _type == 'choice':
+                _index = paramater[name]
+                out_y = {
+                    INDEX: _index,
+                    VALUE: json2paramater(in_x[VALUE][_index], paramater, name=name+"[%d]" % _index)
+                }
+            else:
+                out_y = paramater[name]
+        else:
+            out_y = dict()
+            for key in in_x.keys():
+                out_y[key] = json2paramater(
+                    in_x[key], paramater, name + "[%s]" % str(key))
+    elif isinstance(in_x, list):
+        out_y = list()
+        for i, x_i in enumerate(in_x):
+            out_y.append(json2paramater(x_i, paramater, name + "[%d]" % i))
+    else:
+        logger.info('in_x is not a dict or a list in json2space fuinction %s', str(in_x))
+    return out_y
+
+
+def json2vals(in_x, vals, out_y, name=ROOT):
+    if isinstance(in_x, dict):
+        if TYPE in in_x.keys():
+            _type = in_x[TYPE]
+            name = name + '-' + _type
+
+            try:
+                out_y[name] = vals[INDEX]
+            # TODO - catch exact Exception
+            except Exception:
+                out_y[name] = vals
+
+            if _type == 'choice':
+                _index = vals[INDEX]
+                json2vals(in_x[VALUE][_index], vals[VALUE],
+                          out_y, name=name + "[%d]" % _index)
+        else:
+            for key in in_x.keys():
+                json2vals(in_x[key], vals[key], out_y, name + "[%s]" % str(key))
+    elif isinstance(in_x, list):
+        for i, temp in enumerate(in_x):
+            json2vals(i, vals[temp], out_y, name + "[%d]" % temp)
+
+
+def _split_index(params):
+    result = {}
+    for key in params:
+        if isinstance(params[key], dict):
+            value = params[key][VALUE]
+        else:
+            value = params[key]
+        result[key] = value
+    return result
+
+
+class HyperoptTuner(Tuner):
+    '''
+    HyperoptTuner is a tuner which using hyperopt algorithm.
+    '''
+    def __init__(self, algorithm_name, optimize_mode):
+        self.algorithm_name = algorithm_name
+        self.optimize_mode = OptimizeMode(optimize_mode)
+        self.json = None
+        self.total_data = {}
+        self.rval = None
+
+    def _choose_tuner(self, algorithm_name):
+        if algorithm_name == 'tpe':
+            return hp.tpe.suggest
+        if algorithm_name == 'random_search':
+            return hp.rand.suggest
+        if algorithm_name == 'anneal':
+            return hp.anneal.suggest
+        raise RuntimeError('Not support tuner algorithm in hyperopt.')
+
+    def update_search_space(self, search_space):
+        '''
+        Update search space definition in tuner by search_space in parameters.
+        '''
+        #assert self.json is None
+
+        self.json = search_space
+        search_space_instance = json2space(self.json)
+        rstate = np.random.RandomState()
+        trials = hp.Trials()
+        domain = hp.Domain(None, search_space_instance,
+                           pass_expr_memo_ctrl=None)
+        algorithm = self._choose_tuner(self.algorithm_name)
+        self.rval = hp.FMinIter(algorithm, domain, trials,
+                                max_evals=-1, rstate=rstate, verbose=0)
+        self.rval.catch_eval_exceptions = False
+
+    def generate_parameters(self, parameter_id):
+        """
+		Returns a set of trial (hyper-)parameters, as a serializable object.
+        parameter_id : int
+        """
+        rval = self.rval
+        trials = rval.trials
+        algorithm = rval.algo
+        new_ids = rval.trials.new_trial_ids(1)
+        rval.trials.refresh()
+        random_state = rval.rstate.randint(2**31-1)
+        new_trials = algorithm(new_ids, rval.domain, trials, random_state)
+        rval.trials.refresh()
+        vals = new_trials[0]['misc']['vals']
+        parameter = dict()
+        for key in vals:
+            try:
+                parameter[key] = vals[key][0].item()
+            except Exception:
+                parameter[key] = None
+
+        # remove "_index" from json2parameter and save params-id
+        total_params = json2paramater(self.json, parameter)
+        self.total_data[parameter_id] = total_params
+        params = _split_index(total_params)
+        return params
+
+    def receive_trial_result(self, parameter_id, parameters, reward):
+        '''
+        Record an observation of the objective function
+        parameter_id : int
+        parameters : dict of parameters
+        reward : reward of one trial
+        '''
+        # restore the paramsters contains "_index"
+        if parameter_id not in self.total_data:
+            raise RuntimeError('Received parameter_id not in total_data.')
+        params = self.total_data[parameter_id]
+
+        if self.optimize_mode is OptimizeMode.Maximize:
+            reward = -reward
+
+        rval = self.rval
+        domain = rval.domain
+        trials = rval.trials
+
+        new_id = len(trials)
+
+        rval_specs = [None]
+        rval_results = [domain.new_result()]
+        rval_miscs = [dict(tid=new_id, cmd=domain.cmd, workdir=domain.workdir)]
+
+        vals = params
+        idxs = dict()
+
+        out_y = dict()
+        json2vals(self.json, vals, out_y)
+        vals = out_y
+        for key in domain.params:
+            if key in [VALUE, INDEX]:
+                continue
+            if key not in vals or vals[key] is None or vals[key] == []:
+                idxs[key] = vals[key] = []
+            else:
+                idxs[key] = [new_id]
+                vals[key] = [vals[key]]
+
+        self.miscs_update_idxs_vals(rval_miscs, idxs, vals,
+                                    idxs_map={new_id: new_id},
+                                    assert_all_vals_used=False)
+
+        trial = trials.new_trial_docs([new_id], rval_specs, rval_results, rval_miscs)[0]
+        trial['result'] = {'loss': reward, 'status': 'ok'}
+        trial['state'] = hp.JOB_STATE_DONE
+        trials.insert_trial_docs([trial])
+        trials.refresh()
+
+    def miscs_update_idxs_vals(self, miscs, idxs, vals,
+                               assert_all_vals_used=True,
+                               idxs_map=None):
+        """
+        Unpack the idxs-vals format into the list of dictionaries that is
+        `misc`.
+
+        idxs_map: a dictionary of id->id mappings so that the misc['idxs'] can
+            contain different numbers than the idxs argument. XXX CLARIFY
+        """
+        if idxs_map is None:
+            idxs_map = {}
+
+        assert set(idxs.keys()) == set(vals.keys())
+
+        misc_by_id = {m['tid']: m for m in miscs}
+        for m in miscs:
+            m['idxs'] = dict([(key, []) for key in idxs])
+            m['vals'] = dict([(key, []) for key in idxs])
+
+        for key in idxs:
+            assert len(idxs[key]) == len(vals[key])
+            for tid, val in zip(idxs[key], vals[key]):
+                tid = idxs_map.get(tid, tid)
+                if assert_all_vals_used or tid in misc_by_id:
+                    misc_by_id[tid]['idxs'][key] = [tid]
+                    misc_by_id[tid]['vals'][key] = [val]

src/sdk/pynni/nni/medianstop_assessor/__main__.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import argparse
+import logging
+
+from .medianstop_assessor import MedianstopAssessor
+
+
+logger = logging.getLogger('nni.contrib.medianstop_assessor')
+logger.debug('START')
+
+
+def main():
+    '''
+    main function.
+    '''
+    parser = argparse.ArgumentParser(description='parse command line parameters.')
+    parser.add_argument('--start_from', type=int, default=0, dest='start_step',
+                        help='Assessing each trial from the step start_step.')
+    parser.add_argument('--optimize_mode', type=str, default='maximize',
+                        help='Select optimize mode for Tuner: minimize or maximize.')
+    FLAGS, _ = parser.parse_known_args()
+
+    assessor = MedianstopAssessor(FLAGS.start_step, FLAGS.optimize_mode)
+    assessor.run()
+
+
+try:
+    main()
+except Exception as exception:
+    logger.exception(exception)
+    raise
\ No newline at end of file

src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import logging
+from nni.assessor import Assessor, AssessResult
+
+logger = logging.getLogger('medianstop_Assessor')
+
+class MedianstopAssessor(Assessor):
+    '''
+    MedianstopAssessor is The median stopping rule stops a pending trial X at step S 
+    if the trial’s best objective value by step S is strictly worse than the median value 
+    of the running averages of all completed trials’ objectives reported up to step S
+    '''
+    def __init__(self, start_step, optimize_mode):
+        self.start_step = start_step
+        self.running_history = dict()
+        self.completed_avg_history = dict()
+        if optimize_mode == 'maximize':
+            self.high_better = True
+        elif optimize_mode == 'minimize':
+            self.high_better = False
+        else:
+            self.high_better = True
+            logger.warning('unrecognized optimize_mode', optimize_mode)
+
+    def _update_data(self, trial_job_id, trial_history):
+        if trial_job_id not in self.running_history:
+            self.running_history[trial_job_id] = []
+        self.running_history[trial_job_id].extend(trial_history[len(self.running_history[trial_job_id]):])
+
+    def trial_end(self, trial_job_id, success):
+        '''
+        trial_end
+        '''
+        if trial_job_id in self.running_history:
+            if success:
+                cnt = 0
+                history_sum = 0
+                self.completed_avg_history[trial_job_id] = []
+                for each in self.running_history[trial_job_id]:
+                    cnt += 1
+                    history_sum += each
+                    self.completed_avg_history[trial_job_id].append(history_sum / cnt)
+            self.running_history.pop(trial_job_id)
+        else:
+            logger.warning('trial_end: trial_job_id does not in running_history')
+
+    def assess_trial(self, trial_job_id, trial_history):
+        '''
+        assess_trial
+        '''
+        curr_step = len(trial_history)
+        if curr_step < self.start_step:
+            return AssessResult.Good
+
+        try:
+            num_trial_history = [float(ele) for ele in trial_history]
+        except (TypeError, ValueError) as error:
+            logger.warning('incorrect data type or value:')
+            logger.exception(error)
+        except Exception as error:
+            logger.warning('unrecognized exception:')
+            logger.excpetion(error)
+
+        self._update_data(trial_job_id, num_trial_history)
+        if self.high_better:
+            best_history = max(trial_history)
+        else:
+            best_history = min(trial_history)
+
+        avg_array = []
+        for id in self.completed_avg_history:
+            if len(self.completed_avg_history[id]) >= curr_step:
+                avg_array.append(self.completed_avg_history[id][curr_step - 1])
+        if len(avg_array) > 0:
+            avg_array.sort()
+            if self.high_better:
+                median = avg_array[(len(avg_array)-1) // 2]
+                return AssessResult.Bad if best_history < median else AssessResult.Good
+            else:
+                median = avg_array[len(avg_array) // 2]
+                return AssessResult.Bad if best_history > median else AssessResult.Good
+        else:
+            return AssessResult.Good

src/sdk/pynni/nni/medianstop_assessor/test.py

+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import argparse
+import logging
+import random
+
+from .medianstop_assessor import MedianstopAssessor
+from nni.assessor import AssessResult
+
+
+logger = logging.getLogger('nni.contrib.medianstop_assessor')
+logger.debug('START')
+
+
+def test():
+    '''
+    tests.
+    '''
+    parser = argparse.ArgumentParser(description='parse command line parameters.')
+    parser.add_argument('--start_from', type=int, default=10, dest='start_step',
+                        help='Assessing each trial from the step start_step.')
+    parser.add_argument('--optimize_mode', type=str, default='maximize',
+                        help='Select optimize mode for Tuner: minimize or maximize.')
+    FLAGS, _ = parser.parse_known_args()
+
+    lcs = [[1,1,1,1,1,1,1,1,1,1],
+           [2,2,2,2,2,2,2,2,2,2],
+           [3,3,3,3,3,3,3,3,3,3],
+           [4,4,4,4,4,4,4,4,4,4]]
+    #lcs = [[1,1,1,1,1,1,1,1,1,1],
+    #       [1,1,1,1,1,1,1,1,1,1],
+    #       [1,1,1,1,1,1,1,1,1,1]]
+
+    assessor = MedianstopAssessor(FLAGS.start_step, FLAGS.optimize_mode)
+    for i in range(4):
+        #lc = []
+        to_complete = True
+        for k in range(10):
+            #d = random.randint(i*100+0, i*100+100)
+            #lc.append(d)
+            ret = assessor.assess_trial(i, lcs[i][:k+1])
+            print('result: %d', ret)
+            if ret == AssessResult.Bad:
+                assessor.trial_end(i, False)
+                to_complete = False
+                break
+        if to_complete:
+            assessor.trial_end(i, True)
+
+try:
+    test()
+except Exception as exception:
+    logger.exception(exception)
+    raise
\ No newline at end of file

src/sdk/pynni/nni/platform/__init__.py

+# Copyright (c) Microsoft Corporation. All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+# associated documentation files (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge, publish, distribute,
+# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
+# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# ==================================================================================================
+
+
+# pylint: disable=wildcard-import
+
+from ..common import env_args
+
+if env_args.platform is None:
+    from .standalone import *
+elif env_args.platform == 'unittest':
+    from .test import *
+elif env_args.platform in ('local', 'remote'):
+    from .local import *
+else:
+    raise RuntimeError('Unknown platform %s' % env_args.platform)