Merge pull request #127 from Microsoft/master

merge master

.github/ISSUE_TEMPLATE/bug-report.md

+---
+name: Create an issue
+about: Report an issue or question while using nni instance (deployment).
+
+
+---
+
+<!-- Please use this template while reporting an issue and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!-->
+
+
+**Short summary about the issue/question**:
+
+**Brief what process you are following**: 
+
+<!--deployment related issues
+Please fill this for deployment related issues: 
+- Operating type: Initial deployment / upgrading / operating etc.
+- Brief what deployment process you are following -->
+
+**How to reproduce it**: 
+
+<!--Fill the following information if your issue need diagnostic support from the team, as minimally and precisely as possible!-->
+
+**nni Environment**:
+- nni version:
+- nni mode(local|pai|remote):
+- OS:
+- python version:
+- is conda or virtualenv used?: 
+- is running in docker?:
+
+**Anything else we need to know**:

.github/ISSUE_TEMPLATE/enhancement.md

+---
+name: Enhancement Request
+about: Suggest an enhancement to the nni project
+
+---
+<!-- Please only use this template for submitting enhancement requests -->
+
+
+**What would you like to be added**:
+
+**Why is this needed**:
+
+**Without this feature, how does current nni work**：
+
+**Components that may involve changes**:
+
+**Brief description of your proposal if any**:

docs/en_US/conf.py

@@ -12,9 +12,9 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
+import os
+import sys
+sys.path.insert(0, os.path.abspath('../../src/sdk/pynni'))
 
 from recommonmark.parser import CommonMarkParser
 from recommonmark.transform import AutoStructify

docs/en_US/sdk_reference.rst

@@ -22,10 +22,16 @@ Tuner
 ..  autoclass:: nni.evolution_tuner.evolution_tuner.EvolutionTuner
     :members:
 
+..  autoclass:: nni.smac_tuner.smac_tuner.SMACTuner
+    :members:
+
 ..  autoclass:: nni.gridsearch_tuner.gridsearch_tuner.GridSearchTuner
     :members:
 
-..  autoclass:: nni.smac_tuner.smac_tuner.SMACTuner
+..  autoclass:: nni.networkmorphism_tuner.networkmorphism_tuner.NetworkMorphismTuner
+    :members:
+
+..  autoclass:: nni.metis_tuner.metis_tuner.MetisTuner
     :members:
 
 Assessor
@@ -42,4 +48,5 @@ Assessor
 
 Advisor
 ------------------------
-..  autoclass:: nni.hyperband_advisor.hyperband_advisor.Hyperband
+..  autoclass:: nni.hyperband_advisor.hyperband_advisor.Hyperband
\ No newline at end of file
+    :members:
\ No newline at end of file

docs/requirements.txt

@@ -4,4 +4,10 @@ sphinx-markdown-tables==0.0.9
 sphinx-rtd-theme==0.4.2
 sphinxcontrib-websupport==1.1.0
 recommonmark==0.5.0
-nni==0.5
+hyperopt
\ No newline at end of file
+json_tricks
+numpy
+scipy
+coverage
+git+https://github.com/QuanluZhang/ConfigSpace.git
+git+https://github.com/QuanluZhang/SMAC3.git

src/sdk/pynni/nni/curvefitting_assessor/README.md

@@ -44,20 +44,18 @@ assessor:
     builtinAssessorName: Curvefitting
     classArgs:
       # (required)The total number of epoch.
-      # We need to know the number of epoch to determine which point we need to predict.
+      #  We need to know the number of epoch to determine which point we need to predict.
       epoch_num: 20
       # (optional) choice: maximize, minimize
       * The default value of optimize_mode is maximize
       optimize_mode: maximize
-      # Kindly reminds that if you choose minimize mode, please adjust the value of threshold >= 1.0 (e.g threshold=1.1)
-      # (optional) A trial is determined to be stopped or not
-      # In order to save our computing resource, we start to predict when we have more than start_step(default=6) accuracy points.
-      # only after receiving start_step number of reported intermediate results.
+      # (optional) In order to save our computing resource, we start to predict when we have more than only after receiving start_step number of reported intermediate results.
       * The default value of start_step is 6.
       start_step: 6
       # (optional) The threshold that we decide to early stop the worse performance curve.
       # For example: if threshold = 0.95, optimize_mode = maximize, best performance in the history is 0.9, then we will stop the trial which predict value is lower than 0.95 * 0.9 = 0.855.
       * The default value of threshold is 0.95.
+      # Kindly reminds that if you choose minimize mode, please adjust the value of threshold >= 1.0 (e.g threshold=1.1)
       threshold: 0.95
 ```
 

src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py

@@ -22,11 +22,21 @@ from .model_factory import CurveModel
 logger = logging.getLogger('curvefitting_Assessor')
 
 class CurvefittingAssessor(Assessor):
-    '''
-    CurvefittingAssessor uses learning curve fitting algorithm to predict the learning curve performance in the future.
+    """CurvefittingAssessor uses learning curve fitting algorithm to predict the learning curve performance in the future.
     It stops a pending trial X at step S if the trial's forecast result at target step is convergence and lower than the
     best performance in the history.
-    '''
+
+    Parameters
+    ----------
+    epoch_num: int
+        The total number of epoch
+    optimize_mode: str
+        optimize mode, 'maximize' or 'minimize'
+    start_step: int
+        only after receiving start_step number of reported intermediate results
+    threshold: float
+        The threshold that we decide to early stop the worse performance curve.
+    """
     def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95):
         if start_step <= 0:
             logger.warning('It\'s recommended to set start_step to a positive number')
@@ -51,9 +61,15 @@ class CurvefittingAssessor(Assessor):
         logger.info('Successfully initials the curvefitting assessor')
 
     def trial_end(self, trial_job_id, success):
-        '''
-        trial end: update the best performance of completed trial job
-        '''
+        """update the best performance of completed trial job
+        
+        Parameters
+        ----------
+        trial_job_id: int
+            trial job id
+        success: bool
+            True if succssfully finish the experiment, False otherwise
+        """
         if success:
             if self.set_best_performance:
                 self.completed_best_performance = max(self.completed_best_performance, self.trial_history[-1])
@@ -65,10 +81,25 @@ class CurvefittingAssessor(Assessor):
             logger.info('No need to update, trial job id: ', trial_job_id)
 
     def assess_trial(self, trial_job_id, trial_history):
-        '''
-        assess whether a trial should be early stop by curve fitting algorithm
-        return AssessResult.Good or AssessResult.Bad
-        '''
+        """assess whether a trial should be early stop by curve fitting algorithm
+
+        Parameters
+        ----------
+        trial_job_id: int
+            trial job id
+        trial_history: list
+            The history performance matrix of each trial
+
+        Returns
+        -------
+        bool
+            AssessResult.Good or AssessResult.Bad
+
+        Raises
+        ------
+        Exception
+            unrecognize exception in curvefitting_assessor
+        """
         self.trial_history = trial_history
         curr_step = len(trial_history)
         if curr_step < self.start_step:
@@ -94,4 +125,4 @@ class CurvefittingAssessor(Assessor):
                 return AssessResult.Bad
 
         except Exception as exception:
-            logger.exception('unrecognize exception in curvefitting_asserssor', exception)
+            logger.exception('unrecognize exception in curvefitting_assessor', exception)

src/sdk/pynni/nni/curvefitting_assessor/curvefunctions.py

@@ -25,7 +25,20 @@ curve_combination_models = ['vap', 'pow3', 'linear', 'logx_linear', 'dr_hill_zer
                             'exp4', 'ilog2', 'weibull', 'janoschek']
 
 def vap(x, a, b, c):
-    ''' Vapor pressure model '''
+    """Vapor pressure model
+    
+    Parameters
+    ----------
+    x: int
+    a: float
+    b: float
+    c: float
+
+    Returns
+    -------
+    float
+        np.exp(a+b/x+c*np.log(x))
+    """
     return np.exp(a+b/x+c*np.log(x))
 
 all_models['vap'] = vap
@@ -33,6 +46,20 @@ model_para['vap'] = [-0.622028, -0.470050, 0.042322]
 model_para_num['vap'] = 3
 
 def pow3(x, c, a, alpha):
+    """pow3
+
+    Parameters
+    ----------
+    x: int
+    c: float
+    a: float
+    alpha: float
+
+    Returns
+    -------
+    float
+        c - a * x**(-alpha)
+    """
     return c - a * x**(-alpha)
 
 all_models['pow3'] = pow3
@@ -40,6 +67,19 @@ model_para['pow3'] = [0.84, 0.52, 0.01]
 model_para_num['pow3'] = 3
 
 def linear(x, a, b):
+    """linear
+
+    Parameters
+    ----------
+    x: int
+    a: float
+    b: float
+
+    Returns
+    -------
+    float
+        a*x + b
+    """
     return a*x + b
 
 all_models['linear'] = linear
@@ -47,6 +87,19 @@ model_para['linear'] = [1., 0]
 model_para_num['linear'] = 2
 
 def logx_linear(x, a, b):
+    """logx linear
+
+    Parameters
+    ----------
+    x: int
+    a: float
+    b: float
+
+    Returns
+    -------
+    float
+        a * np.log(x) + b
+    """
     x = np.log(x)
     return a*x + b
 
@@ -55,6 +108,20 @@ model_para['logx_linear'] = [0.378106, 0.046506]
 model_para_num['logx_linear'] = 2
 
 def dr_hill_zero_background(x, theta, eta, kappa):
+    """dr hill zero background
+ 
+    Parameters
+    ----------
+    x: int
+    theta: float
+    eta: float
+    kappa: float
+
+    Returns
+    -------
+    float
+        (theta* x**eta) / (kappa**eta + x**eta)
+    """
     return (theta* x**eta) / (kappa**eta + x**eta)
 
 all_models['dr_hill_zero_background'] = dr_hill_zero_background
@@ -62,7 +129,20 @@ model_para['dr_hill_zero_background'] = [0.772320, 0.586449, 2.460843]
 model_para_num['dr_hill_zero_background'] = 3
 
 def log_power(x, a, b, c):
-    #logistic power
+    """"logistic power
+
+    Parameters
+    ----------
+    x: int
+    a: float
+    b: float
+    c: float
+
+    Returns
+    -------
+    float
+        a/(1.+(x/np.exp(b))**c)
+    """
     return a/(1.+(x/np.exp(b))**c)
 
 all_models['log_power'] = log_power
@@ -70,6 +150,21 @@ model_para['log_power'] = [0.77, 2.98, -0.51]
 model_para_num['log_power'] = 3
 
 def pow4(x, alpha, a, b, c):
+    """pow4
+
+    Parameters
+    ----------
+    x: int
+    alpha: float
+    a: float
+    b: float
+    c: float
+
+    Returns
+    -------
+    float
+        c - (a*x+b)**-alpha
+    """
     return c - (a*x+b)**-alpha
 
 all_models['pow4'] = pow4
@@ -77,10 +172,22 @@ model_para['pow4'] = [0.1, 200, 0., 0.8]
 model_para_num['pow4'] = 4
 
 def mmf(x, alpha, beta, kappa, delta):
-    '''
-    Morgan-Mercer-Flodin
+    """Morgan-Mercer-Flodin
     http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm
-    '''
+
+    Parameters
+    ----------
+    x: int
+    alpha: float
+    beta: float
+    kappa: float
+    delta: float
+
+    Returns
+    -------
+    float
+        alpha - (alpha - beta) / (1. + (kappa * x)**delta)
+    """
     return alpha - (alpha - beta) / (1. + (kappa * x)**delta)
 
 all_models['mmf'] = mmf
@@ -88,6 +195,21 @@ model_para['mmf'] = [0.7, 0.1, 0.01, 5]
 model_para_num['mmf'] = 4
 
 def exp4(x, c, a, b, alpha):
+    """exp4
+
+    Parameters
+    ----------
+    x: int
+    c: float
+    a: float
+    b: float
+    alpha: float
+
+    Returns
+    -------
+    float
+        c - np.exp(-a*(x**alpha)+b)
+    """
     return c - np.exp(-a*(x**alpha)+b)
 
 all_models['exp4'] = exp4
@@ -95,6 +217,19 @@ model_para['exp4'] = [0.7, 0.8, -0.8, 0.3]
 model_para_num['exp4'] = 4
 
 def ilog2(x, c, a):
+    """ilog2
+
+    Parameters
+    ----------
+    x: int
+    c: float
+    a: float
+
+    Returns
+    -------
+    float
+        c - a / np.log(x)
+    """
     return c - a / np.log(x)
 
 all_models['ilog2'] = ilog2
@@ -102,10 +237,22 @@ model_para['ilog2'] = [0.78, 0.43]
 model_para_num['ilog2'] = 2
 
 def weibull(x, alpha, beta, kappa, delta):
-    '''
-    Weibull model
+    """Weibull model
     http://www.pisces-conservation.com/growthhelp/index.html?morgan_mercer_floden.htm
-    '''
+
+    Parameters
+    ----------
+    x: int
+    alpha: float
+    beta: float
+    kappa: float
+    delta: float
+
+    Returns
+    -------
+    float
+        alpha - (alpha - beta) * np.exp(-(kappa * x)**delta)
+    """
     return alpha - (alpha - beta) * np.exp(-(kappa * x)**delta)
 
 all_models['weibull'] = weibull
@@ -113,7 +260,21 @@ model_para['weibull'] = [0.7, 0.1, 0.01, 1]
 model_para_num['weibull'] = 4
 
 def janoschek(x, a, beta, k, delta):
-    '''http://www.pisces-conservation.com/growthhelp/janoschek.htm'''
+    """http://www.pisces-conservation.com/growthhelp/janoschek.htm
+    
+    Parameters
+    ----------
+    x: int
+    a: float
+    beta: float
+    k: float
+    delta: float
+
+    Returns
+    -------
+    float
+        a - (a - beta) * np.exp(-k*x**delta)
+    """
     return a - (a - beta) * np.exp(-k*x**delta)
 
 all_models['janoschek'] = janoschek

src/sdk/pynni/nni/curvefitting_assessor/model_factory.py

@@ -36,6 +36,15 @@ LEAST_FITTED_FUNCTION = 4
 logger = logging.getLogger('curvefitting_Assessor')
 
 class CurveModel(object):
+    """Build a Curve Model to predict the performance
+    
+    Algorithm: https://github.com/Microsoft/nni/blob/master/src/sdk/pynni/nni/curvefitting_assessor/README.md
+
+    Parameters
+    ----------
+    target_pos: int
+        The point we need to predict
+    """
     def __init__(self, target_pos):
         self.target_pos = target_pos
         self.trial_history = []
@@ -45,7 +54,12 @@ class CurveModel(object):
         self.weight_samples = []
 
     def fit_theta(self):
-        '''use least squares to fit all default curves parameter seperately'''
+        """use least squares to fit all default curves parameter seperately
+        
+        Returns
+        -------
+        None
+        """
         x = range(1, self.point_num + 1)
         y = self.trial_history
         for i in range(NUM_OF_FUNCTIONS):
@@ -73,7 +87,12 @@ class CurveModel(object):
                 logger.critical("Exceptions in fit_theta:", exception)
 
     def filter_curve(self):
-        '''filter the poor performing curve'''
+        """filter the poor performing curve
+        
+        Returns
+        -------
+        None
+        """
         avg = np.sum(self.trial_history) / self.point_num
         standard = avg * avg * self.point_num
         predict_data = []
@@ -97,7 +116,20 @@ class CurveModel(object):
         logger.info('List of effective model: ', self.effective_model)
 
     def predict_y(self, model, pos):
-        '''return the predict y of 'model' when epoch = pos'''
+        """return the predict y of 'model' when epoch = pos
+        
+        Parameters
+        ----------
+        model: string
+            name of the curve function model
+        pos: int
+            the epoch number of the position you want to predict
+
+        Returns
+        -------
+        int:
+            The expected matrix at pos
+        """
         if model_para_num[model] == 2:
             y = all_models[model](pos, model_para[model][0], model_para[model][1])
         elif model_para_num[model] == 3:
@@ -107,7 +139,20 @@ class CurveModel(object):
         return y
 
     def f_comb(self, pos, sample):
-        '''return the value of the f_comb when epoch = pos'''
+        """return the value of the f_comb when epoch = pos
+
+        Parameters
+        ----------
+        pos: int
+            the epoch number of the position you want to predict
+        sample: list
+            sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
+
+        Returns
+        -------
+        int
+            The expected matrix at pos with all the active function's prediction
+        """
         ret = 0
         for i in range(self.effective_model_num):
             model = self.effective_model[i]
@@ -116,7 +161,19 @@ class CurveModel(object):
         return ret
 
     def normalize_weight(self, samples):
-        '''normalize weight '''
+        """normalize weight
+        
+        Parameters
+        ----------
+        samples: list
+            a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
+            representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
+
+        Returns
+        -------
+        list
+            samples after normalize weight
+        """
         for i in range(NUM_OF_INSTANCE):
             total = 0
             for j in range(self.effective_model_num):
@@ -126,7 +183,18 @@ class CurveModel(object):
         return samples
 
     def sigma_sq(self, sample):
-        '''returns the value of sigma square, given the weight's sample'''
+        """returns the value of sigma square, given the weight's sample
+        
+        Parameters
+        ----------
+        sample: list
+            sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
+
+        Returns
+        -------
+        float
+            the value of sigma square, given the weight's sample
+        """
         ret = 0
         for i in range(1, self.point_num + 1):
             temp = self.trial_history[i - 1] - self.f_comb(i, sample)
@@ -134,13 +202,37 @@ class CurveModel(object):
         return 1.0 * ret / self.point_num
 
     def normal_distribution(self, pos, sample):
-        '''returns the value of normal distribution, given the weight's sample and target position'''
+        """returns the value of normal distribution, given the weight's sample and target position
+        
+        Parameters
+        ----------
+        pos: int
+            the epoch number of the position you want to predict
+        sample: list
+            sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
+
+        Returns
+        -------
+        float
+            the value of normal distribution
+        """
         curr_sigma_sq = self.sigma_sq(sample)
         delta = self.trial_history[pos - 1] - self.f_comb(pos, sample)
         return np.exp(np.square(delta) / (-2.0 * curr_sigma_sq)) / np.sqrt(2 * np.pi * np.sqrt(curr_sigma_sq))
 
     def likelihood(self, samples):
-        '''likelihood'''
+        """likelihood
+
+        Parameters
+        ----------
+        sample: list
+            sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
+        
+        Returns
+        -------
+        float
+            likelihood
+        """
         ret = np.ones(NUM_OF_INSTANCE)
         for i in range(NUM_OF_INSTANCE):
             for j in range(1, self.point_num + 1):
@@ -148,7 +240,19 @@ class CurveModel(object):
         return ret
 
     def prior(self, samples):
-        '''priori distribution'''
+        """priori distribution
+ 
+        Parameters
+        ----------
+        samples: list
+            a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
+            representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
+        
+        Returns
+        -------
+        float
+            priori distribution
+        """
         ret = np.ones(NUM_OF_INSTANCE)
         for i in range(NUM_OF_INSTANCE):
             for j in range(self.effective_model_num):
@@ -159,7 +263,19 @@ class CurveModel(object):
         return ret
 
     def target_distribution(self, samples):
-        '''posterior probability'''
+        """posterior probability
+        
+        Parameters
+        ----------
+        samples: list
+            a collection of sample, it's a (NUM_OF_INSTANCE * NUM_OF_FUNCTIONS) matrix,
+            representing{{w11, w12, ..., w1k}, {w21, w22, ... w2k}, ...{wk1, wk2,..., wkk}}
+        
+        Returns
+        -------
+        float
+            posterior probability
+        """
         curr_likelihood = self.likelihood(samples)
         curr_prior = self.prior(samples)
         ret = np.ones(NUM_OF_INSTANCE)
@@ -168,8 +284,7 @@ class CurveModel(object):
         return ret
 
     def mcmc_sampling(self):
-        '''
-        Adjust the weight of each function using mcmc sampling.
+        """Adjust the weight of each function using mcmc sampling.
         The initial value of each weight is evenly distribute.
         Brief introduction:
         (1)Definition of sample:
@@ -181,7 +296,11 @@ class CurveModel(object):
             Model is the function we chose right now. Such as: 'wap', 'weibull'.
         (4)Definition of pos:
             Pos is the position we want to predict, corresponds to the value of epoch.
-        '''
+
+        Returns
+        -------
+        None
+        """
         init_weight = np.ones((self.effective_model_num), dtype=np.float) / self.effective_model_num
         self.weight_samples = np.broadcast_to(init_weight, (NUM_OF_INSTANCE, self.effective_model_num))
         for i in range(NUM_OF_SIMULATION_TIME):
@@ -199,7 +318,18 @@ class CurveModel(object):
             self.weight_samples = new_values
 
     def predict(self, trial_history):
-        '''predict the value of target position'''
+        """predict the value of target position
+        
+        Parameters
+        ----------
+        trial_history: list
+            The history performance matrix of each trial.
+
+        Returns
+        -------
+        float
+            expected final result performance of this hyperparameter config
+        """
         self.trial_history = trial_history
         self.point_num = len(trial_history)
         self.fit_theta()

src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py

@@ -17,9 +17,9 @@
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-'''
-hyperband_tuner.py
-'''
+"""
+hyperband_advisor.py
+"""
 
 from enum import Enum, unique
 import sys
@@ -41,24 +41,39 @@ _KEY = 'STEPS'
 
 @unique
 class OptimizeMode(Enum):
-    '''
-    Oprimize Mode class
-    '''
+    """Oprimize Mode class"""
     Minimize = 'minimize'
     Maximize = 'maximize'
 
 def create_parameter_id():
-    '''
-    Create an id
-    '''
+    """Create an id
+    
+    Returns
+    -------
+    int
+        parameter id
+    """
     global _next_parameter_id  # pylint: disable=global-statement
     _next_parameter_id += 1
     return _next_parameter_id - 1
 
 def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-1):
-    '''
-    Create a full id for a specific bracket's hyperparameter configuration
-    '''
+    """Create a full id for a specific bracket's hyperparameter configuration
+    
+    Parameters
+    ----------
+    brackets_id: int
+        brackets id
+    brackets_curr_decay:
+        brackets curr decay
+    increased_id: int
+        increased id
+
+    Returns
+    -------
+    int
+        params id
+    """
     if increased_id == -1:
         increased_id = str(create_parameter_id())
     params_id = '_'.join([str(brackets_id),
@@ -67,11 +82,20 @@ def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-
     return params_id
 
 def json2paramater(ss_spec, random_state):
-    '''
-    Randomly generate values for hyperparameters from hyperparameter space i.e., x.
-    ss_spec: hyperparameter space
-    random_state: random operator to generate random values
-    '''
+    """Randomly generate values for hyperparameters from hyperparameter space i.e., x.
+    
+    Parameters
+    ----------
+    ss_spec:
+        hyperparameter space
+    random_state:
+        random operator to generate random values
+
+    Returns
+    -------
+    Parameter:
+        Parameters in this experiment
+    """
     if isinstance(ss_spec, dict):
         if '_type' in ss_spec.keys():
             _type = ss_spec['_type']
@@ -95,9 +119,24 @@ def json2paramater(ss_spec, random_state):
     return chosen_params
 
 class Bracket():
-    '''
-    A bracket in Hyperband, all the information of a bracket is managed by an instance of this class
-    '''
+    """A bracket in Hyperband, all the information of a bracket is managed by an instance of this class
+    
+    Parameters
+    ----------
+    s: int
+        The current SH iteration index.
+    s_max: int
+        total number of SH iterations
+    eta: float
+        In each iteration, a complete run of sequential halving is executed. In it,
+		after evaluating each configuration on the same subset size, only a fraction of
+		1/eta of them 'advances' to the next round.
+    R:
+        the budget associated with each stage
+    optimize_mode: str
+        optimize mode, 'maximize' or 'minimize'
+    """
+    
     def __init__(self, s, s_max, eta, R, optimize_mode):
         self.bracket_id = s
         self.s_max = s_max
@@ -113,33 +152,37 @@ class Bracket():
         self.no_more_trial = False
 
     def is_completed(self):
-        '''
-        check whether this bracket has sent out all the hyperparameter configurations
-        '''
+        """check whether this bracket has sent out all the hyperparameter configurations"""
         return self.no_more_trial
 
     def get_n_r(self):
-        '''
-        return the values of n and r for the next round
-        '''
+        """return the values of n and r for the next round"""
         return math.floor(self.n / self.eta**self.i), self.r * self.eta**self.i
 
     def increase_i(self):
-        '''
-        i means the ith round. Increase i by 1
-        '''
+        """i means the ith round. Increase i by 1"""
         self.i += 1
         if self.i > self.bracket_id:
             self.no_more_trial = True
 
     def set_config_perf(self, i, parameter_id, seq, value):
-        '''
-        update trial's latest result with its sequence number, e.g., epoch number or batch number
-        i: the ith round
-        parameter_id: the id of the trial/parameter
-        seq: sequence number, e.g., epoch number or batch number
-        value: latest result with sequence number seq
-        '''
+        """update trial's latest result with its sequence number, e.g., epoch number or batch number
+        
+        Parameters
+        ----------
+        i: int
+            the ith round
+        parameter_id: int
+            the id of the trial/parameter
+        seq: int
+            sequence number, e.g., epoch number or batch number
+        value: int
+            latest result with sequence number seq
+
+        Returns
+        -------
+        None
+        """
         if parameter_id in self.configs_perf[i]:
             if self.configs_perf[i][parameter_id][0] < seq:
                 self.configs_perf[i][parameter_id] = [seq, value]
@@ -148,10 +191,14 @@ class Bracket():
             
 
     def inform_trial_end(self, i):
-        '''
-        If the trial is finished and the corresponding round (i.e., i) has all its trials finished,
+        """If the trial is finished and the corresponding round (i.e., i) has all its trials finished,
         it will choose the top k trials for the next round (i.e., i+1)
-        '''
+
+        Parameters
+        ----------
+        i: int
+            the ith round
+        """
         global _KEY # pylint: disable=global-statement
         self.num_finished_configs[i] += 1
         _logger.debug('bracket id: %d, round: %d %d, finished: %d, all: %d', self.bracket_id, self.i, i, self.num_finished_configs[i], self.num_configs_to_run[i])
@@ -181,10 +228,18 @@ class Bracket():
         return None
 
     def get_hyperparameter_configurations(self, num, r, searchspace_json, random_state): # pylint: disable=invalid-name
-        '''
-        Randomly generate num hyperparameter configurations from search space
-        num: the number of hyperparameter configurations
-        '''
+        """Randomly generate num hyperparameter configurations from search space
+
+        Parameters
+        ----------
+        num: int
+            the number of hyperparameter configurations
+        
+        Returns
+        -------
+        list
+            a list of hyperparameter configurations. Format: [[key1, value1], [key2, value2], ...]
+        """
         global _KEY # pylint: disable=global-statement
         assert self.i == 0
         hyperparameter_configs = dict()
@@ -197,11 +252,15 @@ class Bracket():
         return [[key, value] for key, value in hyperparameter_configs.items()]
 
     def _record_hyper_configs(self, hyper_configs):
-        '''
-        after generating one round of hyperconfigs, this function records the generated hyperconfigs,
+        """after generating one round of hyperconfigs, this function records the generated hyperconfigs,
         creates a dict to record the performance when those hyperconifgs are running, set the number of finished configs
         in this round to be 0, and increase the round number.
-        '''
+
+        Parameters
+        ----------
+        hyper_configs: list
+            the generated hyperconfigs
+        """
         self.hyper_configs.append(hyper_configs)
         self.configs_perf.append(dict())
         self.num_finished_configs.append(0)
@@ -209,6 +268,13 @@ class Bracket():
         self.increase_i()
 
 def extract_scalar_reward(value, scalar_key='default'):
+    """
+    Raises
+    ------
+    RuntimeError
+        Incorrect final result: the final result should be float/int,
+        or a dict which has a key named "default" whose value is float/int.
+    """
     if isinstance(value, float) or isinstance(value, int):
         reward = value
     elif isinstance(value, dict) and scalar_key in value and isinstance(value[scalar_key], (float, int)):
@@ -218,18 +284,21 @@ def extract_scalar_reward(value, scalar_key='default'):
     return reward
 
 class Hyperband(MsgDispatcherBase):
-    '''
-    Hyperband inherit from MsgDispatcherBase rather than Tuner,
-    because it integrates both tuner's functions and assessor's functions.
+    """Hyperband inherit from MsgDispatcherBase rather than Tuner, because it integrates both tuner's functions and assessor's functions.
     This is an implementation that could fully leverage available resources, i.e., high parallelism.
     A single execution of Hyperband takes a finite budget of (s_max + 1)B.
-    '''
+
+    Parameters
+    ----------
+    R: int
+        the maximum amount of resource that can be allocated to a single configuration
+    eta: int
+        the variable that controls the proportion of configurations discarded in each round of SuccessiveHalving
+    optimize_mode: str
+        optimize mode, 'maximize' or 'minimize'
+    """
     def __init__(self, R, eta=3, optimize_mode='maximize'):
-        '''
-        R: the maximum amount of resource that can be allocated to a single configuration
-        eta: the variable that controls the proportion of configurations discarded in each round of SuccessiveHalving
-        B = (s_max + 1)R
-        '''
+        """B = (s_max + 1)R"""
         super()
         self.R = R                        # pylint: disable=invalid-name
         self.eta = eta
@@ -254,26 +323,31 @@ class Hyperband(MsgDispatcherBase):
         pass
 
     def handle_initialize(self, data):
-        '''
-        data is search space
-        '''
+        """data is search space
+
+        Parameters
+        ----------
+        data: int
+            number of trial jobs
+        """
         self.handle_update_search_space(data)
         send(CommandType.Initialized, '')
         return True
 
     def handle_request_trial_jobs(self, data):
-        '''
-        data: number of trial jobs
-        '''
+        """
+        Parameters
+        ----------
+        data: int
+            number of trial jobs
+        """
         for _ in range(data):
             self._request_one_trial_job()
 
         return True
 
     def _request_one_trial_job(self):
-        '''
-        get one trial job, i.e., one hyperparameter configuration.
-        '''
+        """get one trial job, i.e., one hyperparameter configuration."""
         if not self.generated_hyper_configs:
             if self.curr_s < 0:
                 # have tried all configurations
@@ -308,21 +382,28 @@ class Hyperband(MsgDispatcherBase):
         return True
 
     def handle_update_search_space(self, data):
-        '''
-        data: JSON object, which is search space
-        '''
+        """data: JSON object, which is search space
+        
+        Parameters
+        ----------
+        data: int
+            number of trial jobs
+        """
         self.searchspace_json = data
         self.random_state = np.random.RandomState()
 
         return True
 
     def handle_trial_end(self, data):
-        '''
-        data: it has three keys: trial_job_id, event, hyper_params
+        """
+        Parameters
+        ----------
+        data: dict()
+            it has three keys: trial_job_id, event, hyper_params
             trial_job_id: the id generated by training service
             event: the job's state
             hyper_params: the hyperparameters (a string) generated and returned by tuner
-        '''
+        """
         hyper_params = json_tricks.loads(data['hyper_params'])
         bracket_id, i, _ = hyper_params['parameter_id'].split('_')
         hyper_configs = self.brackets[int(bracket_id)].inform_trial_end(int(i))
@@ -344,9 +425,17 @@ class Hyperband(MsgDispatcherBase):
         return True
 
     def handle_report_metric_data(self, data):
-        '''
-        data: it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'.
-        '''
+        """
+        Parameters
+        ----------
+        data: 
+            it is an object which has keys 'parameter_id', 'value', 'trial_job_id', 'type', 'sequence'.
+        
+        Raises
+        ------
+        ValueError
+            Data type not supported
+        """
         value = extract_scalar_reward(data['value'])
         bracket_id, i, _ = data['parameter_id'].split('_')
         bracket_id = int(bracket_id)

src/sdk/pynni/nni/medianstop_assessor/medianstop_assessor.py

@@ -21,11 +21,17 @@ from nni.assessor import Assessor, AssessResult
 logger = logging.getLogger('medianstop_Assessor')
 
 class MedianstopAssessor(Assessor):
-    '''
-    MedianstopAssessor is The median stopping rule stops a pending trial X at step S 
+    """MedianstopAssessor is The median stopping rule stops a pending trial X at step S 
     if the trial’s best objective value by step S is strictly worse than the median value 
     of the running averages of all completed trials’ objectives reported up to step S
-    '''
+    
+    Parameters
+    ----------
+    optimize_mode: str
+        optimize mode, 'maximize' or 'minimize'
+    start_step: int
+        only after receiving start_step number of reported intermediate results
+    """
     def __init__(self, optimize_mode='maximize', start_step=0):
         self.start_step = start_step
         self.running_history = dict()
@@ -39,14 +45,29 @@ class MedianstopAssessor(Assessor):
             logger.warning('unrecognized optimize_mode', optimize_mode)
 
     def _update_data(self, trial_job_id, trial_history):
+        """update data
+
+        Parameters
+        ----------
+        trial_job_id: int
+            trial job id
+        trial_history: list
+            The history performance matrix of each trial
+        """
         if trial_job_id not in self.running_history:
             self.running_history[trial_job_id] = []
         self.running_history[trial_job_id].extend(trial_history[len(self.running_history[trial_job_id]):])
 
     def trial_end(self, trial_job_id, success):
-        '''
-        trial_end
-        '''
+        """trial_end
+        
+        Parameters
+        ----------
+        trial_job_id: int
+            trial job id
+        success: bool
+            True if succssfully finish the experiment, False otherwise
+        """
         if trial_job_id in self.running_history:
             if success:
                 cnt = 0
@@ -61,9 +82,25 @@ class MedianstopAssessor(Assessor):
             logger.warning('trial_end: trial_job_id does not in running_history')
 
     def assess_trial(self, trial_job_id, trial_history):
-        '''
-        assess_trial
-        '''
+        """assess_trial
+        
+        Parameters
+        ----------
+        trial_job_id: int
+            trial job id
+        trial_history: list
+            The history performance matrix of each trial
+
+        Returns
+        -------
+        bool
+            AssessResult.Good or AssessResult.Bad
+
+        Raises
+        ------
+        Exception
+            unrecognize exception in medianstop_assessor
+        """
         curr_step = len(trial_history)
         if curr_step < self.start_step:
             return AssessResult.Good
@@ -74,7 +111,7 @@ class MedianstopAssessor(Assessor):
             logger.warning('incorrect data type or value:')
             logger.exception(error)
         except Exception as error:
-            logger.warning('unrecognized exception:')
+            logger.warning('unrecognized exception in medianstop_assessor:')
             logger.excpetion(error)
 
         self._update_data(trial_job_id, num_trial_history)

src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py

@@ -23,7 +23,18 @@ import json
 import numpy as np
 
 def get_json_content(file_path):
-    '''Load json file content'''
+    """Load json file content
+    
+    Parameters
+    ----------
+    file_path:
+        path to the file
+    
+    Raises
+    ------
+    TypeError
+        Error with the file path
+    """
     try:
         with open(file_path, 'r') as file:
             return json.load(file)
@@ -32,15 +43,34 @@ def get_json_content(file_path):
         return None
 
 def generate_pcs(nni_search_space_content):
-    '''
+    """Generate the Parameter Configuration Space (PCS) which defines the 
+    legal ranges of the parameters to be optimized and their default values.
+    
+    Generally, the format is:
     # parameter_name categorical {value_1, ..., value_N} [default value]
     # parameter_name ordinal {value_1, ..., value_N} [default value]
     # parameter_name integer [min_value, max_value] [default value]
     # parameter_name integer [min_value, max_value] [default value] log
     # parameter_name real [min_value, max_value] [default value]
     # parameter_name real [min_value, max_value] [default value] log
-    # https://automl.github.io/SMAC3/stable/options.html
-    '''
+    
+    Reference: https://automl.github.io/SMAC3/stable/options.html
+
+    Parameters
+    ----------
+    nni_search_space_content: search_space
+        The search space in this experiment in nni
+    
+    Returns
+    -------
+    Parameter Configuration Space (PCS)
+        the legal ranges of the parameters to be optimized and their default values
+
+    Raises
+    ------
+    RuntimeError
+        unsupported type or value error or incorrect search space
+    """
     categorical_dict = {}
     search_space = nni_search_space_content
     with open('param_config_space.pcs', 'w') as pcs_fd:
@@ -92,34 +122,85 @@ def generate_pcs(nni_search_space_content):
     return None
 
 def generate_scenario(ss_content):
-    '''
-    # deterministic, 1/0
-    # output_dir,
-    # paramfile, 
-    # run_obj, 'quality'
+    """Generate the scenario. The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and 
+    can be constructed either by providing an actual scenario-object, or by specifing the options in a scenario file.
+    
+    Reference: https://automl.github.io/SMAC3/stable/options.html
+
+    The format of the scenario file is one option per line:
+    OPTION1 = VALUE1
+    OPTION2 = VALUE2
+    ...
+
+    Parameters
+    ----------
+    abort_on_first_run_crash: bool
+        If true, SMAC will abort if the first run of the target algorithm crashes. Default: True, 
+        because trials reported to nni tuner would always in success state
+    algo: function
+        Specifies the target algorithm call that SMAC will optimize. Interpreted as a bash-command.
+        Not required by tuner, but required by nni's training service for running trials
+    always_race_default:
+        Race new incumbents always against default configuration
+    cost_for_crash:
+        Defines the cost-value for crashed runs on scenarios with quality as run-obj. Default: 2147483647.0.
+        Trials reported to nni tuner would always in success state
+    cutoff_time:
+        Maximum runtime, after which the target algorithm is cancelled. `Required if *run_obj* is runtime`
+    deterministic: bool
+        If true, the optimization process will be repeatable.
+    execdir:
+        Specifies the path to the execution-directory. Default: .
+        Trials are executed by nni's training service
+    feature_file:
+        Specifies the file with the instance-features.
+        No features specified or feature file is not supported
+    initial_incumbent:
+        DEFAULT is the default from the PCS. Default: DEFAULT. Must be from: [‘DEFAULT’, ‘RANDOM’].
+    input_psmac_dirs:
+        For parallel SMAC, multiple output-directories are used.
+        Parallelism is supported by nni
+    instance_file:
+        Specifies the file with the training-instances. Not supported
+    intensification_percentage:
+        The fraction of time to be used on intensification (versus choice of next Configurations). Default: 0.5.
+        Not supported, trials are controlled by nni's training service and kill be assessor
+    maxR: int
+        Maximum number of calls per configuration. Default: 2000.
+    memory_limit:
+        Maximum available memory the target algorithm can occupy before being cancelled.
+    minR: int
+        Minimum number of calls per configuration. Default: 1.
+    output_dir:
+        Specifies the output-directory for all emerging files, such as logging and results.
+        Default: smac3-output_2018-01-22_15:05:56_807070.
+    overall_obj:
+    	PARX, where X is an integer defining the penalty imposed on timeouts (i.e. runtimes that exceed the cutoff-time).
+        Timeout is not supported
+    paramfile:
+        Specifies the path to the PCS-file.
+    run_obj:
+        Defines what metric to optimize. When optimizing runtime, cutoff_time is required as well.
+        Must be from: [‘runtime’, ‘quality’].
+    runcount_limit: int
+        Maximum number of algorithm-calls during optimization. Default: inf.
+        Use default because this is controlled by nni
+    shared_model:
+        Whether to run SMAC in parallel mode. Parallelism is supported by nni
+    test_instance_file:
+        Specifies the file with the test-instances. Instance is not supported
+    tuner-timeout:
+        Maximum amount of CPU-time used for optimization. Not supported
+    wallclock_limit: int
+        Maximum amount of wallclock-time used for optimization. Default: inf.
+        Use default because this is controlled by nni
 
-    # the following keys use default value or empty
-    # algo, not required by tuner, but required by nni's training service for running trials
-    # abort_on_first_run_crash, because trials reported to nni tuner would always in success state
-    # always_race_default,
-    # cost_for_crash, trials reported to nni tuner would always in success state
-    # cutoff_time, 
-    # execdir, trials are executed by nni's training service
-    # feature_file, no features specified or feature file is not supported
-    # initial_incumbent, use default value
-    # input_psmac_dirs, parallelism is supported by nni
-    # instance_file, not supported
-    # intensification_percentage, not supported, trials are controlled by nni's training service and kill be assessor
-    # maxR, use default, 2000
-    # minR, use default, 1
-    # overall_obj, timeout is not supported
-    # shared_model, parallelism is supported by nni
-    # test_instance_file, instance is not supported
-    # tuner-timeout, not supported
-    # runcount_limit, default: inf., use default because this is controlled by nni
-    # wallclock_limit,default: inf., use default because this is controlled by nni
-    # please refer to https://automl.github.io/SMAC3/stable/options.html
-    '''
+    Returns
+    -------
+    Scenario:
+        The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and can be constructed
+        either by providing an actual scenario-object, or by specifing the options in a scenario file
+    """
     with open('scenario.txt', 'w') as sce_fd:
         sce_fd.write('deterministic = 0\n')
         #sce_fd.write('output_dir = \n')

src/sdk/pynni/nni/smac_tuner/smac_tuner.py

@@ -17,9 +17,9 @@
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-'''
+"""
 smac_tuner.py
-'''
+"""
 
 from nni.tuner import Tuner
 
@@ -38,17 +38,19 @@ from smac.facade.epils_facade import EPILS
 
 @unique
 class OptimizeMode(Enum):
-    '''
-    Oprimize Mode class
-    '''
+    """Oprimize Mode class"""
     Minimize = 'minimize'
     Maximize = 'maximize'
 
 class SMACTuner(Tuner):
+    """
+    Parameters
+    ----------
+    optimize_mode: str
+        optimize mode, 'maximize' or 'minimize'
+    """
     def __init__(self, optimize_mode):
-        '''
-        Constructor
-        '''
+        """Constructor"""
         self.logger = logging.getLogger(
             self.__module__ + "." + self.__class__.__name__)
         self.optimize_mode = OptimizeMode(optimize_mode)
@@ -61,9 +63,13 @@ class SMACTuner(Tuner):
         self.categorical_dict = {}
 
     def _main_cli(self):
-        '''
-        Main function of SMAC for CLI interface
-        '''
+        """Main function of SMAC for CLI interface
+        
+        Returns
+        -------
+        instance
+            optimizer
+        """
         self.logger.info("SMAC call: %s" % (" ".join(sys.argv)))
 
         cmd_reader = CMDReader()
@@ -123,11 +129,15 @@ class SMACTuner(Tuner):
         return optimizer
 
     def update_search_space(self, search_space):
-        '''
-        TODO: this is urgly, we put all the initialization work in this method, 
-        because initialization relies on search space, also because update_search_space is called at the beginning.
+        """TODO: this is urgly, we put all the initialization work in this method, because initialization relies
+        on search space, also because update_search_space is called at the beginning.
         NOTE: updating search space is not supported.
-        '''
+
+        Parameters
+        ----------
+        search_space:
+            search space
+        """
         if not self.update_ss_done:
             self.categorical_dict = generate_scenario(search_space)
             if self.categorical_dict is None:
@@ -140,9 +150,22 @@ class SMACTuner(Tuner):
             self.logger.warning('update search space is not supported.')
 
     def receive_trial_result(self, parameter_id, parameters, value):
-        '''
-        receive_trial_result
-        '''
+        """receive_trial_result
+       
+        Parameters
+        ----------
+        parameter_id: int
+            parameter id
+        parameters:
+            parameters
+        value:
+            value
+        
+        Raises
+        ------
+        RuntimeError
+            Received parameter id not in total_data
+        """
         reward = self.extract_scalar_reward(value)
         if self.optimize_mode is OptimizeMode.Maximize:
             reward = -reward
@@ -156,12 +179,21 @@ class SMACTuner(Tuner):
             self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward)
 
     def convert_loguniform_categorical(self, challenger_dict):
-        '''
-        Convert the values of type `loguniform` back to their initial range
+        """Convert the values of type `loguniform` back to their initial range
         Also, we convert categorical:
         categorical values in search space are changed to list of numbers before,
         those original values will be changed back in this function
-        '''
+        
+        Parameters
+        ----------
+        challenger_dict: dict
+            challenger dict
+
+        Returns
+        -------
+        dict
+            challenger dict
+        """
         for key, value in challenger_dict.items():
             # convert to loguniform
             if key in self.loguniform_key:
@@ -173,9 +205,18 @@ class SMACTuner(Tuner):
         return challenger_dict
 
     def generate_parameters(self, parameter_id):
-        '''
-        generate one instance of hyperparameters
-        '''
+        """generate one instance of hyperparameters
+        
+        Parameters
+        ----------
+        parameter_id: int
+            parameter id
+        
+        Returns
+        -------
+        list
+            new generated parameters
+        """
         if self.first_one:
             init_challenger = self.smbo_solver.nni_smac_start()
             self.total_data[parameter_id] = init_challenger
@@ -189,11 +230,18 @@ class SMACTuner(Tuner):
                 return self.convert_loguniform_categorical(challenger.get_dictionary())
 
     def generate_multiple_parameters(self, parameter_id_list):
-        '''
-        generate mutiple instances of hyperparameters
-        '''
+        """generate mutiple instances of hyperparameters
         
-
+        Parameters
+        ----------
+        parameter_id_list: list
+            list of parameter id
+        
+        Returns
+        -------
+        list
+            list of new generated parameters
+        """
         if self.first_one:
             params = []
             for one_id in parameter_id_list: