Skip to content

GitLab

Unverified Commit 8c203f30 authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Merge pull request #211 from microsoft/master 

merge master
parents 7c1ab114 483232c8
Hide whitespace changes
Inline Side-by-side
Showing with 191 additions and 239 deletions
src/sdk/pynni/nni/metis_tuner/Regression_GMM/Selection.py
View file @ 8c203f30
...@@ -24,7 +24,6 @@ import sys ...@@ -24,7 +24,6 @@ import sys
import nni.metis_tuner.lib_acquisition_function as lib_acquisition_function import nni.metis_tuner.lib_acquisition_function as lib_acquisition_function
import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation
import nni.metis_tuner.lib_data as lib_data
sys.path.insert(1, os.path.join(sys.path[0], '..')) sys.path.insert(1, os.path.join(sys.path[0], '..'))
...@@ -52,13 +51,13 @@ def selection_r(x_bounds, ...@@ -52,13 +51,13 @@ def selection_r(x_bounds,
Select using different types. Select using different types.
''' '''
minimize_starting_points = clusteringmodel_gmm_good.sample(n_samples=num_starting_points) minimize_starting_points = clusteringmodel_gmm_good.sample(n_samples=num_starting_points)
outputs = selection(x_bounds, x_types, outputs = selection(x_bounds, x_types,
clusteringmodel_gmm_good, clusteringmodel_gmm_good,
clusteringmodel_gmm_bad, clusteringmodel_gmm_bad,
minimize_starting_points[0], minimize_starting_points[0],
minimize_constraints_fun) minimize_constraints_fun)
return outputs return outputs
def selection(x_bounds, def selection(x_bounds,
......
src/sdk/pynni/nni/metis_tuner/Regression_GP/OutlierDetection.py
View file @ 8c203f30
...@@ -19,12 +19,12 @@ ...@@ -19,12 +19,12 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import argparse, json, os, sys import os
import sys
from multiprocessing.dummy import Pool as ThreadPool from multiprocessing.dummy import Pool as ThreadPool
import nni.metis_tuner.Regression_GP.CreateModel as gp_create_model import nni.metis_tuner.Regression_GP.CreateModel as gp_create_model
import nni.metis_tuner.Regression_GP.Prediction as gp_prediction import nni.metis_tuner.Regression_GP.Prediction as gp_prediction
import nni.metis_tuner.lib_data as lib_data
sys.path.insert(1, os.path.join(sys.path[0], '..')) sys.path.insert(1, os.path.join(sys.path[0], '..'))
...@@ -71,14 +71,15 @@ def outlierDetection_threaded(samples_x, samples_y_aggregation): ...@@ -71,14 +71,15 @@ def outlierDetection_threaded(samples_x, samples_y_aggregation):
else: else:
print("error here.") print("error here.")
outliers = None if len(outliers) == 0 else outliers outliers = outliers if outliers else None
return outliers return outliers
def outlierDetection(samples_x, samples_y_aggregation): def outlierDetection(samples_x, samples_y_aggregation):
''' '''
TODO
''' '''
outliers = [] outliers = []
for samples_idx in range(0, len(samples_x)): for samples_idx, _ in enumerate(samples_x):
#sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n" #sys.stderr.write("[%s] DEBUG: Evaluating %d of %d samples\n"
# \ % (os.path.basename(__file__), samples_idx + 1, len(samples_x))) # \ % (os.path.basename(__file__), samples_idx + 1, len(samples_x)))
diagnostic_regressor_gp = gp_create_model.create_model(\ diagnostic_regressor_gp = gp_create_model.create_model(\
...@@ -93,5 +94,5 @@ def outlierDetection(samples_x, samples_y_aggregation): ...@@ -93,5 +94,5 @@ def outlierDetection(samples_x, samples_y_aggregation):
"expected_sigma": sigma, "expected_sigma": sigma,
"difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)}) "difference": abs(samples_y_aggregation[samples_idx] - mu) - (2.33 * sigma)})
outliers = None if len(outliers) == 0 else outliers outliers = outliers if outliers else None
return outliers return outliers
src/sdk/pynni/nni/metis_tuner/lib_data.py
View file @ 8c203f30
...@@ -31,6 +31,7 @@ def match_val_type(vals, vals_bounds, vals_types): ...@@ -31,6 +31,7 @@ def match_val_type(vals, vals_bounds, vals_types):
for i, _ in enumerate(vals_types): for i, _ in enumerate(vals_types):
if vals_types[i] == "discrete_int": if vals_types[i] == "discrete_int":
# Find the closest integer in the array, vals_bounds # Find the closest integer in the array, vals_bounds
# pylint: disable=cell-var-from-loop
vals_new.append(min(vals_bounds[i], key=lambda x: abs(x - vals[i]))) vals_new.append(min(vals_bounds[i], key=lambda x: abs(x - vals[i])))
elif vals_types[i] == "range_int": elif vals_types[i] == "range_int":
# Round down to the nearest integer # Round down to the nearest integer
...@@ -64,4 +65,3 @@ def rand(x_bounds, x_types): ...@@ -64,4 +65,3 @@ def rand(x_bounds, x_types):
return None return None
return outputs return outputs
\ No newline at end of file
src/sdk/pynni/nni/metis_tuner/metis_tuner.py
View file @ 8c203f30
...@@ -20,14 +20,11 @@ ...@@ -20,14 +20,11 @@
import copy import copy
import logging import logging
import numpy as np
import os
import random import random
import statistics import statistics
import sys
import warnings import warnings
from enum import Enum, unique
from multiprocessing.dummy import Pool as ThreadPool from multiprocessing.dummy import Pool as ThreadPool
import numpy as np
import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation import nni.metis_tuner.lib_constraint_summation as lib_constraint_summation
import nni.metis_tuner.lib_data as lib_data import nni.metis_tuner.lib_data as lib_data
...@@ -99,6 +96,8 @@ class MetisTuner(Tuner): ...@@ -99,6 +96,8 @@ class MetisTuner(Tuner):
self.minimize_constraints_fun = None self.minimize_constraints_fun = None
self.minimize_starting_points = None self.minimize_starting_points = None
self.supplement_data_num = 0 self.supplement_data_num = 0
self.x_bounds = []
self.x_types = []
def update_search_space(self, search_space): def update_search_space(self, search_space):
...@@ -144,7 +143,7 @@ class MetisTuner(Tuner): ...@@ -144,7 +143,7 @@ class MetisTuner(Tuner):
self.x_types[idx] = 'discrete_int' self.x_types[idx] = 'discrete_int'
else: else:
logger.info("Metis Tuner doesn't support this kind of variable: " + str(key_type)) logger.info("Metis Tuner doesn't support this kind of variable: %s", key_type)
raise RuntimeError("Metis Tuner doesn't support this kind of variable: " + str(key_type)) raise RuntimeError("Metis Tuner doesn't support this kind of variable: " + str(key_type))
else: else:
logger.info("The format of search space is not a dict.") logger.info("The format of search space is not a dict.")
...@@ -198,7 +197,7 @@ class MetisTuner(Tuner): ...@@ -198,7 +197,7 @@ class MetisTuner(Tuner):
minimize_starting_points=self.minimize_starting_points, minimize_starting_points=self.minimize_starting_points,
minimize_constraints_fun=self.minimize_constraints_fun) minimize_constraints_fun=self.minimize_constraints_fun)
logger.info("Generate paramageters:\n" + str(results)) logger.info("Generate paramageters:\n%s", results)
return results return results
...@@ -217,8 +216,8 @@ class MetisTuner(Tuner): ...@@ -217,8 +216,8 @@ class MetisTuner(Tuner):
value = -value value = -value
logger.info("Received trial result.") logger.info("Received trial result.")
logger.info("value is :" + str(value)) logger.info("value is :%s", value)
logger.info("parameter is : " + str(parameters)) logger.info("parameter is : %s", parameters)
# parse parameter to sample_x # parse parameter to sample_x
sample_x = [0 for i in range(len(self.key_order))] sample_x = [0 for i in range(len(self.key_order))]
...@@ -271,10 +270,12 @@ class MetisTuner(Tuner): ...@@ -271,10 +270,12 @@ class MetisTuner(Tuner):
minimize_constraints_fun=minimize_constraints_fun) minimize_constraints_fun=minimize_constraints_fun)
if not lm_current: if not lm_current:
return None return None
logger.info({'hyperparameter': lm_current['hyperparameter'], logger.info({
'expected_mu': lm_current['expected_mu'], 'hyperparameter': lm_current['hyperparameter'],
'expected_sigma': lm_current['expected_sigma'], 'expected_mu': lm_current['expected_mu'],
'reason': "exploitation_gp"}) 'expected_sigma': lm_current['expected_sigma'],
'reason': "exploitation_gp"
})
if no_candidates is False: if no_candidates is False:
# ===== STEP 2: Get recommended configurations for exploration ===== # ===== STEP 2: Get recommended configurations for exploration =====
...@@ -289,10 +290,12 @@ class MetisTuner(Tuner): ...@@ -289,10 +290,12 @@ class MetisTuner(Tuner):
if results_exploration is not None: if results_exploration is not None:
if _num_past_samples(results_exploration['hyperparameter'], samples_x, samples_y) == 0: if _num_past_samples(results_exploration['hyperparameter'], samples_x, samples_y) == 0:
temp_candidate = {'hyperparameter': results_exploration['hyperparameter'], temp_candidate = {
'expected_mu': results_exploration['expected_mu'], 'hyperparameter': results_exploration['hyperparameter'],
'expected_sigma': results_exploration['expected_sigma'], 'expected_mu': results_exploration['expected_mu'],
'reason': "exploration"} 'expected_sigma': results_exploration['expected_sigma'],
'reason': "exploration"
}
candidates.append(temp_candidate) candidates.append(temp_candidate)
logger.info("DEBUG: 1 exploration candidate selected\n") logger.info("DEBUG: 1 exploration candidate selected\n")
...@@ -322,11 +325,14 @@ class MetisTuner(Tuner): ...@@ -322,11 +325,14 @@ class MetisTuner(Tuner):
if results_exploitation is not None: if results_exploitation is not None:
if _num_past_samples(results_exploitation['hyperparameter'], samples_x, samples_y) == 0: if _num_past_samples(results_exploitation['hyperparameter'], samples_x, samples_y) == 0:
temp_expected_mu, temp_expected_sigma = gp_prediction.predict(results_exploitation['hyperparameter'], gp_model['model']) temp_expected_mu, temp_expected_sigma = \
temp_candidate = {'hyperparameter': results_exploitation['hyperparameter'], gp_prediction.predict(results_exploitation['hyperparameter'], gp_model['model'])
'expected_mu': temp_expected_mu, temp_candidate = {
'expected_sigma': temp_expected_sigma, 'hyperparameter': results_exploitation['hyperparameter'],
'reason': "exploitation_gmm"} 'expected_mu': temp_expected_mu,
'expected_sigma': temp_expected_sigma,
'reason': "exploitation_gmm"
}
candidates.append(temp_candidate) candidates.append(temp_candidate)
logger.info("DEBUG: 1 exploitation_gmm candidate selected\n") logger.info("DEBUG: 1 exploitation_gmm candidate selected\n")
...@@ -349,7 +355,7 @@ class MetisTuner(Tuner): ...@@ -349,7 +355,7 @@ class MetisTuner(Tuner):
results_outliers = gp_outlier_detection.outlierDetection_threaded(samples_x, samples_y_aggregation) results_outliers = gp_outlier_detection.outlierDetection_threaded(samples_x, samples_y_aggregation)
if results_outliers is not None: if results_outliers is not None:
for results_outlier in results_outliers: for results_outlier in results_outliers: # pylint: disable=not-an-iterable
if _num_past_samples(samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x: if _num_past_samples(samples_x[results_outlier['samples_idx']], samples_x, samples_y) < max_resampling_per_x:
temp_candidate = {'hyperparameter': samples_x[results_outlier['samples_idx']],\ temp_candidate = {'hyperparameter': samples_x[results_outlier['samples_idx']],\
'expected_mu': results_outlier['expected_mu'],\ 'expected_mu': results_outlier['expected_mu'],\
...@@ -398,10 +404,12 @@ class MetisTuner(Tuner): ...@@ -398,10 +404,12 @@ class MetisTuner(Tuner):
next_candidate = {'hyperparameter': next_candidate, 'reason': "random", next_candidate = {'hyperparameter': next_candidate, 'reason': "random",
'expected_mu': expected_mu, 'expected_sigma': expected_sigma} 'expected_mu': expected_mu, 'expected_sigma': expected_sigma}
# ===== STEP 7: If current optimal hyperparameter occurs in the history or exploration probability is less than the threshold, take next config as exploration step ===== # ===== STEP 7 =====
# If current optimal hyperparameter occurs in the history or exploration probability is less than the threshold,
# take next config as exploration step
outputs = self._pack_output(lm_current['hyperparameter']) outputs = self._pack_output(lm_current['hyperparameter'])
ap = random.uniform(0, 1) ap = random.uniform(0, 1)
if outputs in self.total_data or ap<=self.exploration_probability: if outputs in self.total_data or ap <= self.exploration_probability:
if next_candidate is not None: if next_candidate is not None:
outputs = self._pack_output(next_candidate['hyperparameter']) outputs = self._pack_output(next_candidate['hyperparameter'])
else: else:
...@@ -419,14 +427,14 @@ class MetisTuner(Tuner): ...@@ -419,14 +427,14 @@ class MetisTuner(Tuner):
""" """
_completed_num = 0 _completed_num = 0
for trial_info in data: for trial_info in data:
logger.info("Importing data, current processing progress %s / %s" %(_completed_num, len(data))) logger.info("Importing data, current processing progress %s / %s", _completed_num, len(data))
_completed_num += 1 _completed_num += 1
assert "parameter" in trial_info assert "parameter" in trial_info
_params = trial_info["parameter"] _params = trial_info["parameter"]
assert "value" in trial_info assert "value" in trial_info
_value = trial_info['value'] _value = trial_info['value']
if not _value: if not _value:
logger.info("Useless trial data, value is %s, skip this trial data." %_value) logger.info("Useless trial data, value is %s, skip this trial data.", _value)
continue continue
self.supplement_data_num += 1 self.supplement_data_num += 1
_parameter_id = '_'.join(["ImportData", str(self.supplement_data_num)]) _parameter_id = '_'.join(["ImportData", str(self.supplement_data_num)])
......
src/sdk/pynni/nni/msg_dispatcher.py
View file @ 8c203f30
...@@ -61,7 +61,7 @@ _customized_parameter_ids = set() ...@@ -61,7 +61,7 @@ _customized_parameter_ids = set()
def _create_parameter_id(): def _create_parameter_id():
global _next_parameter_id # pylint: disable=global-statement global _next_parameter_id
_next_parameter_id += 1 _next_parameter_id += 1
return _next_parameter_id - 1 return _next_parameter_id - 1
...@@ -106,15 +106,15 @@ class MsgDispatcher(MsgDispatcherBase): ...@@ -106,15 +106,15 @@ class MsgDispatcher(MsgDispatcherBase):
self.tuner.update_search_space(data) self.tuner.update_search_space(data)
send(CommandType.Initialized, '') send(CommandType.Initialized, '')
def send_trial_callback(self, id, params): def send_trial_callback(self, id_, params):
"""For tuner to issue trial config when the config is generated """For tuner to issue trial config when the config is generated
""" """
send(CommandType.NewTrialJob, _pack_parameter(id, params)) send(CommandType.NewTrialJob, _pack_parameter(id_, params))
def handle_request_trial_jobs(self, data): def handle_request_trial_jobs(self, data):
# data: number or trial jobs # data: number or trial jobs
ids = [_create_parameter_id() for _ in range(data)] ids = [_create_parameter_id() for _ in range(data)]
_logger.debug("requesting for generating params of {}".format(ids)) _logger.debug("requesting for generating params of %s", ids)
params_list = self.tuner.generate_multiple_parameters(ids, st_callback=self.send_trial_callback) params_list = self.tuner.generate_multiple_parameters(ids, st_callback=self.send_trial_callback)
for i, _ in enumerate(params_list): for i, _ in enumerate(params_list):
...@@ -218,7 +218,8 @@ class MsgDispatcher(MsgDispatcherBase): ...@@ -218,7 +218,8 @@ class MsgDispatcher(MsgDispatcherBase):
try: try:
result = self.assessor.assess_trial(trial_job_id, ordered_history) result = self.assessor.assess_trial(trial_job_id, ordered_history)
except Exception as e: except Exception as e:
_logger.exception('Assessor error') _logger.error('Assessor error')
_logger.exception(e)
if isinstance(result, bool): if isinstance(result, bool):
result = AssessResult.Good if result else AssessResult.Bad result = AssessResult.Good if result else AssessResult.Bad
......
src/sdk/pynni/nni/msg_dispatcher_base.py
View file @ 8c203f30
...@@ -18,7 +18,6 @@ ...@@ -18,7 +18,6 @@
# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# ================================================================================================== # ==================================================================================================
import os
import threading import threading
import logging import logging
from multiprocessing.dummy import Pool as ThreadPool from multiprocessing.dummy import Pool as ThreadPool
...@@ -146,7 +145,7 @@ class MsgDispatcherBase(Recoverable): ...@@ -146,7 +145,7 @@ class MsgDispatcherBase(Recoverable):
pass pass
def process_command(self, command, data): def process_command(self, command, data):
_logger.debug('process_command: command: [{}], data: [{}]'.format(command, data)) _logger.debug('process_command: command: [%s], data: [%s]', command, data)
command_handlers = { command_handlers = {
# Tuner commands: # Tuner commands:
......
src/sdk/pynni/nni/nas_utils.py
View file @ 8c203f30
...@@ -25,6 +25,11 @@ from . import trial ...@@ -25,6 +25,11 @@ from . import trial
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
_MUTABLE_LAYER_SPACE_PREFIX = "_mutable_layer" _MUTABLE_LAYER_SPACE_PREFIX = "_mutable_layer"
_namespace = {}
_tf_variables = {}
_arch_logits_list = []
_optimizer = None
_train_op = None
def classic_mode( def classic_mode(
...@@ -64,47 +69,40 @@ def enas_mode( ...@@ -64,47 +69,40 @@ def enas_mode(
it can be known which inputs should be masked and which op should be executed.''' it can be known which inputs should be masked and which op should be executed.'''
name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) name_prefix = "{}_{}".format(mutable_id, mutable_layer_id)
# store namespace # store namespace
if 'name_space' not in globals(): _namespace[mutable_id] = True
global name_space _namespace[name_prefix] = dict()
name_space = dict() _namespace[name_prefix]['funcs'] = list(funcs)
name_space[mutable_id] = True _namespace[name_prefix]['optional_inputs'] = list(optional_inputs)
name_space[name_prefix] = dict()
name_space[name_prefix]['funcs'] = list(funcs)
name_space[name_prefix]['optional_inputs'] = list(optional_inputs)
# create tensorflow variables as 1/0 signals used to form subgraph # create tensorflow variables as 1/0 signals used to form subgraph
if 'tf_variables' not in globals():
global tf_variables
tf_variables = dict()
name_for_optional_inputs = name_prefix + '_optional_inputs' name_for_optional_inputs = name_prefix + '_optional_inputs'
name_for_funcs = name_prefix + '_funcs' name_for_funcs = name_prefix + '_funcs'
tf_variables[name_prefix] = dict() _tf_variables[name_prefix] = dict()
tf_variables[name_prefix]['optional_inputs'] = tf.get_variable(name_for_optional_inputs, _tf_variables[name_prefix]['optional_inputs'] = tf.get_variable(
[len( name_for_optional_inputs,
optional_inputs)], [len(optional_inputs)],
dtype=tf.bool, dtype=tf.bool,
trainable=False) trainable=False
tf_variables[name_prefix]['funcs'] = tf.get_variable( )
_tf_variables[name_prefix]['funcs'] = tf.get_variable(
name_for_funcs, [], dtype=tf.int64, trainable=False) name_for_funcs, [], dtype=tf.int64, trainable=False)
# get real values using their variable names # get real values using their variable names
real_optional_inputs_value = [optional_inputs[name] real_optional_inputs_value = [optional_inputs[name]
for name in name_space[name_prefix]['optional_inputs']] for name in _namespace[name_prefix]['optional_inputs']]
real_func_value = [funcs[name] real_func_value = [funcs[name]
for name in name_space[name_prefix]['funcs']] for name in _namespace[name_prefix]['funcs']]
real_funcs_args = [funcs_args[name] real_funcs_args = [funcs_args[name]
for name in name_space[name_prefix]['funcs']] for name in _namespace[name_prefix]['funcs']]
# build tensorflow graph of geting chosen inputs by masking # build tensorflow graph of geting chosen inputs by masking
real_chosen_inputs = tf.boolean_mask( real_chosen_inputs = tf.boolean_mask(
real_optional_inputs_value, tf_variables[name_prefix]['optional_inputs']) real_optional_inputs_value, _tf_variables[name_prefix]['optional_inputs'])
# build tensorflow graph of different branches by using tf.case # build tensorflow graph of different branches by using tf.case
branches = dict() branches = dict()
func_output = None
for func_id in range(len(funcs)): for func_id in range(len(funcs)):
func_output = real_func_value[func_id]( func_output = real_func_value[func_id]([fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id])
[fixed_inputs, real_chosen_inputs], **real_funcs_args[func_id]) branches[tf.equal(_tf_variables[name_prefix]['funcs'], func_id)] = lambda: func_output
branches[tf.equal(tf_variables[name_prefix]['funcs'], layer_out = tf.case(branches, exclusive=True, default=lambda: func_output)
func_id)] = lambda: func_output
layer_out = tf.case(branches, exclusive=True,
default=lambda: func_output)
return layer_out return layer_out
...@@ -157,12 +155,9 @@ def darts_mode( ...@@ -157,12 +155,9 @@ def darts_mode(
layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name]) layer_outs = [func([fixed_inputs, optional_inputs], **funcs_args[func_name])
for func_name, func in funcs.items()] for func_name, func in funcs.items()]
# Create architecture weights for every func(op) # Create architecture weights for every func(op)
var_name = "{}_{}_".format(mutable_id, mutable_layer_id, "arch_weights") var_name = "{}_{}_arch_weights".format(mutable_id, mutable_layer_id)
if 'arch_logits_list' not in globals(): arch_logits = tf.get_variable(var_name, shape=[len(funcs)], trainable=False)
global arch_logits_list _arch_logits_list.append(arch_logits)
arch_logits_list = list()
arch_logits = tf.get_variable(var_name, shape=[len[funcs]], trainable=False)
arch_logits_list.append(arch_logits)
arch_weights = tf.nn.softmax(arch_logits) arch_weights = tf.nn.softmax(arch_logits)
layer_out = tf.add_n([arch_weights[idx] * out for idx, out in enumerate(layer_outs)]) layer_out = tf.add_n([arch_weights[idx] * out for idx, out in enumerate(layer_outs)])
...@@ -186,19 +181,19 @@ def reload_tensorflow_variables(tf, session): ...@@ -186,19 +181,19 @@ def reload_tensorflow_variables(tf, session):
mutable_layers.add((mutable_id, mutable_layer_id)) mutable_layers.add((mutable_id, mutable_layer_id))
mutable_layers = sorted(list(mutable_layers)) mutable_layers = sorted(list(mutable_layers))
for mutable_id, mutable_layer_id in mutable_layers: for mutable_id, mutable_layer_id in mutable_layers:
if mutable_id not in name_space: if mutable_id not in _namespace:
_logger.warning("{} not found in name space".format(mutable_id)) _logger.warning("%s not found in name space", mutable_id)
continue continue
name_prefix = "{}_{}".format(mutable_id, mutable_layer_id) name_prefix = "{}_{}".format(mutable_id, mutable_layer_id)
# get optional inputs names # get optional inputs names
optional_inputs = name_space[name_prefix]['optional_inputs'] optional_inputs = _namespace[name_prefix]['optional_inputs']
# extract layer information from the subgraph sampled by tuner # extract layer information from the subgraph sampled by tuner
chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs) chosen_layer, chosen_inputs = _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inputs)
chosen_layer = name_space[name_prefix]['funcs'].index(chosen_layer) chosen_layer = _namespace[name_prefix]['funcs'].index(chosen_layer)
chosen_inputs = [1 if inp in chosen_inputs else 0 for inp in optional_inputs] chosen_inputs = [1 if inp in chosen_inputs else 0 for inp in optional_inputs]
# load these information into pre-defined tensorflow variables # load these information into pre-defined tensorflow variables
tf_variables[name_prefix]['funcs'].load(chosen_layer, session) _tf_variables[name_prefix]['funcs'].load(chosen_layer, session)
tf_variables[name_prefix]['optional_inputs'].load( _tf_variables[name_prefix]['optional_inputs'].load(
chosen_inputs, session) chosen_inputs, session)
...@@ -218,15 +213,13 @@ def _decompose_general_key(key): ...@@ -218,15 +213,13 @@ def _decompose_general_key(key):
def darts_training(tf, session, loss, feed_dict): def darts_training(tf, session, loss, feed_dict):
if 'optimizer' not in globals(): global _optimizer, _train_op
global arch_logits_list if _optimizer is None:
global optimizer _optimizer = tf.MomentumOptimizer(learning_rate=0.025)
global train_op
optimizer = tf.MomentumOptimizer(learning_rate=0.025)
# TODO: Calculate loss # TODO: Calculate loss
grads_and_vars = optimizer.compute_gradients(loss, arch_logits_list) grads_and_vars = _optimizer.compute_gradients(loss, _arch_logits_list)
train_op = optimizer.apply_gradients(grads_and_vars) _train_op = _optimizer.apply_gradients(grads_and_vars)
session.run(train_op) session.run(_train_op)
def training_update(nas_mode, tf=None, session=None, loss=None, feed_dict=None): def training_update(nas_mode, tf=None, session=None, loss=None, feed_dict=None):
...@@ -258,12 +251,11 @@ def _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inpu ...@@ -258,12 +251,11 @@ def _get_layer_and_inputs_from_tuner(mutable_id, mutable_layer_id, optional_inpu
chosen_inputs = [] chosen_inputs = []
# make sure dict -> list produce stable result by sorting # make sure dict -> list produce stable result by sorting
optional_inputs_keys = sorted(optional_inputs) optional_inputs_keys = sorted(optional_inputs)
for i in range(optional_input_size): for _ in range(optional_input_size):
chosen_inputs.append(optional_inputs_keys[optional_input_state % len(optional_inputs)]) chosen_inputs.append(optional_inputs_keys[optional_input_state % len(optional_inputs)])
optional_input_state //= len(optional_inputs) optional_input_state //= len(optional_inputs)
_logger.info("%s_%s: layer: %s, optional inputs: %s" % (mutable_id, mutable_layer_id, _logger.info("%s_%s: layer: %s, optional inputs: %s", mutable_id, mutable_layer_id, chosen_layer, chosen_inputs)
chosen_layer, chosen_inputs))
return chosen_layer, chosen_inputs return chosen_layer, chosen_inputs
...@@ -278,12 +270,12 @@ def convert_nas_search_space(search_space): ...@@ -278,12 +270,12 @@ def convert_nas_search_space(search_space):
if "_type" not in v: if "_type" not in v:
# this should not happen # this should not happen
_logger.warning("There is no _type in one of your search space values with key '%s'" _logger.warning("There is no _type in one of your search space values with key '%s'"
". Please check your search space" % k) ". Please check your search space", k)
ret[k] = v ret[k] = v
elif v["_type"] != "mutable_layer": elif v["_type"] != "mutable_layer":
ret[k] = v ret[k] = v
else: else:
_logger.info("Converting mutable_layer search space with key '%s'" % k) _logger.info("Converting mutable_layer search space with key '%s'", k)
# v["_value"] looks like {'mutable_layer_1': {'layer_choice': ...} ...} # v["_value"] looks like {'mutable_layer_1': {'layer_choice': ...} ...}
values = v["_value"] values = v["_value"]
for layer_name, layer_data in values.items(): for layer_name, layer_data in values.items():
...@@ -305,13 +297,13 @@ def convert_nas_search_space(search_space): ...@@ -305,13 +297,13 @@ def convert_nas_search_space(search_space):
_logger.error("Might not be able to handle optional_input_size < 0, please double check") _logger.error("Might not be able to handle optional_input_size < 0, please double check")
input_size[1] += 1 input_size[1] += 1
else: else:
_logger.info("Optional input choices are set to empty by default in %s" % layer_key) _logger.info("Optional input choices are set to empty by default in %s", layer_key)
input_size = [0, 1] input_size = [0, 1]
if layer_data.get("optional_inputs"): if layer_data.get("optional_inputs"):
total_state_size = len(layer_data["optional_inputs"]) ** (input_size[1] - 1) total_state_size = len(layer_data["optional_inputs"]) ** (input_size[1] - 1)
else: else:
_logger.info("Optional inputs not found in %s" % layer_key) _logger.info("Optional inputs not found in %s", layer_key)
total_state_size = 1 total_state_size = 1
converted = { converted = {
......
src/sdk/pynni/nni/networkmorphism_tuner/graph.py
View file @ 8c203f30
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
# ================================================================================================== # ==================================================================================================
import json import json
from collections import Iterable from collections.abc import Iterable
from copy import deepcopy, copy from copy import deepcopy, copy
from queue import Queue from queue import Queue
...@@ -653,7 +653,7 @@ class Graph: ...@@ -653,7 +653,7 @@ class Graph:
return JSONModel(self).data return JSONModel(self).data
@classmethod @classmethod
def parsing_json_model(self, json_model): def parsing_json_model(cls, json_model):
'''build a graph from json '''build a graph from json
''' '''
return json_to_graph(json_model) return json_to_graph(json_model)
...@@ -910,7 +910,6 @@ def graph_to_onnx(graph, onnx_model_path): ...@@ -910,7 +910,6 @@ def graph_to_onnx(graph, onnx_model_path):
def onnx_to_graph(onnx_model, input_shape): def onnx_to_graph(onnx_model, input_shape):
import onnx
# to do in the future using onnx ir # to do in the future using onnx ir
graph = Graph(input_shape, False) graph = Graph(input_shape, False)
graph.parsing_onnx_model(onnx_model) graph.parsing_onnx_model(onnx_model)
......
src/sdk/pynni/nni/networkmorphism_tuner/layer_transformer.py
View file @ 8c203f30
...@@ -124,7 +124,7 @@ def wider_pre_conv(layer, n_add_filters, weighted=True): ...@@ -124,7 +124,7 @@ def wider_pre_conv(layer, n_add_filters, weighted=True):
student_w = teacher_w.copy() student_w = teacher_w.copy()
student_b = teacher_b.copy() student_b = teacher_b.copy()
# target layer update (i) # target layer update (i)
for i in range(len(rand)): for i, _ in enumerate(rand):
teacher_index = rand[i] teacher_index = rand[i]
new_weight = teacher_w[teacher_index, ...] new_weight = teacher_w[teacher_index, ...]
new_weight = new_weight[np.newaxis, ...] new_weight = new_weight[np.newaxis, ...]
......
src/sdk/pynni/nni/networkmorphism_tuner/layers.py
View file @ 8c203f30
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
# ================================================================================================== # ==================================================================================================
from abc import abstractmethod from abc import abstractmethod
from collections import Iterable from collections.abc import Iterable
import torch import torch
from torch import nn from torch import nn
...@@ -76,7 +76,6 @@ class StubLayer: ...@@ -76,7 +76,6 @@ class StubLayer:
def build(self, shape): def build(self, shape):
'''build shape. '''build shape.
''' '''
pass
def set_weights(self, weights): def set_weights(self, weights):
'''set weights. '''set weights.
...@@ -86,22 +85,18 @@ class StubLayer: ...@@ -86,22 +85,18 @@ class StubLayer:
def import_weights(self, torch_layer): def import_weights(self, torch_layer):
'''import weights. '''import weights.
''' '''
pass
def import_weights_keras(self, keras_layer): def import_weights_keras(self, keras_layer):
'''import weights from keras layer. '''import weights from keras layer.
''' '''
pass
def export_weights(self, torch_layer): def export_weights(self, torch_layer):
'''export weights. '''export weights.
''' '''
pass
def export_weights_keras(self, keras_layer): def export_weights_keras(self, keras_layer):
'''export weights to keras layer. '''export weights to keras layer.
''' '''
pass
def get_weights(self): def get_weights(self):
'''get weights. '''get weights.
...@@ -122,7 +117,6 @@ class StubLayer: ...@@ -122,7 +117,6 @@ class StubLayer:
def to_real_layer(self): def to_real_layer(self):
'''to real layer. '''to real layer.
''' '''
pass
def __str__(self): def __str__(self):
'''str() function to print. '''str() function to print.
...@@ -576,6 +570,7 @@ def to_real_keras_layer(layer): ...@@ -576,6 +570,7 @@ def to_real_keras_layer(layer):
return layers.Flatten() return layers.Flatten()
if is_layer(layer, "GlobalAveragePooling"): if is_layer(layer, "GlobalAveragePooling"):
return layers.GlobalAveragePooling2D() return layers.GlobalAveragePooling2D()
return None # note: this is not written by original author, feel free to modify if you think it's incorrect
def is_layer(layer, layer_type): def is_layer(layer, layer_type):
...@@ -608,6 +603,7 @@ def is_layer(layer, layer_type): ...@@ -608,6 +603,7 @@ def is_layer(layer, layer_type):
return isinstance(layer, (StubFlatten,)) return isinstance(layer, (StubFlatten,))
elif layer_type == "GlobalAveragePooling": elif layer_type == "GlobalAveragePooling":
return isinstance(layer, StubGlobalPooling) return isinstance(layer, StubGlobalPooling)
return None # note: this is not written by original author, feel free to modify if you think it's incorrect
def layer_description_extractor(layer, node_to_id): def layer_description_extractor(layer, node_to_id):
...@@ -664,7 +660,6 @@ def layer_description_extractor(layer, node_to_id): ...@@ -664,7 +660,6 @@ def layer_description_extractor(layer, node_to_id):
def layer_description_builder(layer_information, id_to_node): def layer_description_builder(layer_information, id_to_node):
'''build layer from description. '''build layer from description.
''' '''
# pylint: disable=W0123
layer_type = layer_information[0] layer_type = layer_information[0]
layer_input_ids = layer_information[1] layer_input_ids = layer_information[1]
...@@ -678,26 +673,26 @@ def layer_description_builder(layer_information, id_to_node): ...@@ -678,26 +673,26 @@ def layer_description_builder(layer_information, id_to_node):
filters = layer_information[4] filters = layer_information[4]
kernel_size = layer_information[5] kernel_size = layer_information[5]
stride = layer_information[6] stride = layer_information[6]
return eval(layer_type)( return globals()[layer_type](
input_channel, filters, kernel_size, stride, layer_input, layer_output input_channel, filters, kernel_size, stride, layer_input, layer_output
) )
elif layer_type.startswith("StubDense"): elif layer_type.startswith("StubDense"):
input_units = layer_information[3] input_units = layer_information[3]
units = layer_information[4] units = layer_information[4]
return eval(layer_type)(input_units, units, layer_input, layer_output) return globals()[layer_type](input_units, units, layer_input, layer_output)
elif layer_type.startswith("StubBatchNormalization"): elif layer_type.startswith("StubBatchNormalization"):
num_features = layer_information[3] num_features = layer_information[3]
return eval(layer_type)(num_features, layer_input, layer_output) return globals()[layer_type](num_features, layer_input, layer_output)
elif layer_type.startswith("StubDropout"): elif layer_type.startswith("StubDropout"):
rate = layer_information[3] rate = layer_information[3]
return eval(layer_type)(rate, layer_input, layer_output) return globals()[layer_type](rate, layer_input, layer_output)
elif layer_type.startswith("StubPooling"): elif layer_type.startswith("StubPooling"):
kernel_size = layer_information[3] kernel_size = layer_information[3]
stride = layer_information[4] stride = layer_information[4]
padding = layer_information[5] padding = layer_information[5]
return eval(layer_type)(kernel_size, stride, padding, layer_input, layer_output) return globals()[layer_type](kernel_size, stride, padding, layer_input, layer_output)
else: else:
return eval(layer_type)(layer_input, layer_output) return globals()[layer_type](layer_input, layer_output)
def layer_width(layer): def layer_width(layer):
......
src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py
View file @ 8c203f30
...@@ -310,4 +310,3 @@ class NetworkMorphismTuner(Tuner): ...@@ -310,4 +310,3 @@ class NetworkMorphismTuner(Tuner):
def import_data(self, data): def import_data(self, data):
pass pass
src/sdk/pynni/nni/platform/__init__.py
View file @ 8c203f30
...@@ -19,8 +19,6 @@ ...@@ -19,8 +19,6 @@
# ================================================================================================== # ==================================================================================================
# pylint: disable=wildcard-import
from ..env_vars import trial_env_vars from ..env_vars import trial_env_vars
if trial_env_vars.NNI_PLATFORM is None: if trial_env_vars.NNI_PLATFORM is None:
......
src/sdk/pynni/nni/platform/local.py
View file @ 8c203f30
...@@ -39,8 +39,8 @@ if not os.path.exists(_outputdir): ...@@ -39,8 +39,8 @@ if not os.path.exists(_outputdir):
_nni_platform = trial_env_vars.NNI_PLATFORM _nni_platform = trial_env_vars.NNI_PLATFORM
if _nni_platform == 'local': if _nni_platform == 'local':
_log_file_path = os.path.join(_outputdir, 'trial.log') _log_file_path = os.path.join(_outputdir, 'trial.log')
init_logger(_log_file_path) init_logger(_log_file_path)
_multiphase = trial_env_vars.MULTI_PHASE _multiphase = trial_env_vars.MULTI_PHASE
...@@ -58,7 +58,7 @@ def request_next_parameter(): ...@@ -58,7 +58,7 @@ def request_next_parameter():
def get_next_parameter(): def get_next_parameter():
global _param_index global _param_index
params_file_name = '' params_file_name = ''
if _multiphase and (_multiphase == 'true' or _multiphase == 'True'): if _multiphase in ('true', 'True'):
params_file_name = ('parameter_{}.cfg'.format(_param_index), 'parameter.cfg')[_param_index == 0] params_file_name = ('parameter_{}.cfg'.format(_param_index), 'parameter.cfg')[_param_index == 0]
else: else:
if _param_index > 0: if _param_index > 0:
...@@ -92,7 +92,7 @@ def send_metric(string): ...@@ -92,7 +92,7 @@ def send_metric(string):
file = open(_metric_file.name) file = open(_metric_file.name)
file.close() file.close()
else: else:
subprocess.run(['touch', _metric_file.name], check = True) subprocess.run(['touch', _metric_file.name], check=True)
def get_experiment_id(): def get_experiment_id():
return trial_env_vars.NNI_EXP_ID return trial_env_vars.NNI_EXP_ID
......
src/sdk/pynni/nni/protocol.py
View file @ 8c203f30
...@@ -21,7 +21,6 @@ ...@@ -21,7 +21,6 @@
import logging import logging
import threading import threading
from enum import Enum from enum import Enum
from .common import multi_thread_enabled
class CommandType(Enum): class CommandType(Enum):
...@@ -49,7 +48,6 @@ try: ...@@ -49,7 +48,6 @@ try:
_out_file = open(4, 'wb') _out_file = open(4, 'wb')
except OSError: except OSError:
_msg = 'IPC pipeline not exists, maybe you are importing tuner/assessor from trial code?' _msg = 'IPC pipeline not exists, maybe you are importing tuner/assessor from trial code?'
import logging
logging.getLogger(__name__).warning(_msg) logging.getLogger(__name__).warning(_msg)
...@@ -64,7 +62,7 @@ def send(command, data): ...@@ -64,7 +62,7 @@ def send(command, data):
data = data.encode('utf8') data = data.encode('utf8')
assert len(data) < 1000000, 'Command too long' assert len(data) < 1000000, 'Command too long'
msg = b'%b%06d%b' % (command.value, len(data), data) msg = b'%b%06d%b' % (command.value, len(data), data)
logging.getLogger(__name__).debug('Sending command, data: [%s]' % msg) logging.getLogger(__name__).debug('Sending command, data: [%s]', msg)
_out_file.write(msg) _out_file.write(msg)
_out_file.flush() _out_file.flush()
finally: finally:
...@@ -76,7 +74,7 @@ def receive(): ...@@ -76,7 +74,7 @@ def receive():
Returns a tuple of command (CommandType) and payload (str) Returns a tuple of command (CommandType) and payload (str)
""" """
header = _in_file.read(8) header = _in_file.read(8)
logging.getLogger(__name__).debug('Received command, header: [%s]' % header) logging.getLogger(__name__).debug('Received command, header: [%s]', header)
if header is None or len(header) < 8: if header is None or len(header) < 8:
# Pipe EOF encountered # Pipe EOF encountered
logging.getLogger(__name__).debug('Pipe EOF encountered') logging.getLogger(__name__).debug('Pipe EOF encountered')
...@@ -85,5 +83,5 @@ def receive(): ...@@ -85,5 +83,5 @@ def receive():
data = _in_file.read(length) data = _in_file.read(length)
command = CommandType(header[:2]) command = CommandType(header[:2])
data = data.decode('utf8') data = data.decode('utf8')
logging.getLogger(__name__).debug('Received command, data: [%s]' % data) logging.getLogger(__name__).debug('Received command, data: [%s]', data)
return command, data return command, data
src/sdk/pynni/nni/recoverable.py
View file @ 8c203f30
...@@ -21,6 +21,7 @@ ...@@ -21,6 +21,7 @@
import os import os
class Recoverable: class Recoverable:
def load_checkpoint(self): def load_checkpoint(self):
pass pass
...@@ -31,4 +32,4 @@ class Recoverable: ...@@ -31,4 +32,4 @@ class Recoverable:
ckp_path = os.getenv('NNI_CHECKPOINT_DIRECTORY') ckp_path = os.getenv('NNI_CHECKPOINT_DIRECTORY')
if ckp_path is not None and os.path.isdir(ckp_path): if ckp_path is not None and os.path.isdir(ckp_path):
return ckp_path return ckp_path
return None return None
\ No newline at end of file
src/sdk/pynni/nni/smac_tuner/__init__.py
View file @ 8c203f30
from .smac_tuner import SMACTuner
\ No newline at end of file
src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py
View file @ 8c203f30
...@@ -18,18 +18,17 @@ ...@@ -18,18 +18,17 @@
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import os
import json import json
import numpy as np import numpy as np
def get_json_content(file_path): def get_json_content(file_path):
"""Load json file content """Load json file content
Parameters Parameters
---------- ----------
file_path: file_path:
path to the file path to the file
Raises Raises
------ ------
TypeError TypeError
...@@ -42,10 +41,10 @@ def get_json_content(file_path): ...@@ -42,10 +41,10 @@ def get_json_content(file_path):
print('Error: ', err) print('Error: ', err)
return None return None
def generate_pcs(nni_search_space_content): def generate_pcs(nni_search_space_content):
"""Generate the Parameter Configuration Space (PCS) which defines the """Generate the Parameter Configuration Space (PCS) which defines the
legal ranges of the parameters to be optimized and their default values. legal ranges of the parameters to be optimized and their default values.
Generally, the format is: Generally, the format is:
# parameter_name categorical {value_1, ..., value_N} [default value] # parameter_name categorical {value_1, ..., value_N} [default value]
# parameter_name ordinal {value_1, ..., value_N} [default value] # parameter_name ordinal {value_1, ..., value_N} [default value]
...@@ -53,19 +52,15 @@ def generate_pcs(nni_search_space_content): ...@@ -53,19 +52,15 @@ def generate_pcs(nni_search_space_content):
# parameter_name integer [min_value, max_value] [default value] log # parameter_name integer [min_value, max_value] [default value] log
# parameter_name real [min_value, max_value] [default value] # parameter_name real [min_value, max_value] [default value]
# parameter_name real [min_value, max_value] [default value] log # parameter_name real [min_value, max_value] [default value] log
Reference: https://automl.github.io/SMAC3/stable/options.html Reference: https://automl.github.io/SMAC3/stable/options.html
Parameters Parameters
---------- ----------
nni_search_space_content: search_space nni_search_space_content: search_space
The search space in this experiment in nni The search space in this experiment in nni
Returns Returns
------- -------
Parameter Configuration Space (PCS) Parameter Configuration Space (PCS)
the legal ranges of the parameters to be optimized and their default values the legal ranges of the parameters to be optimized and their default values
Raises Raises
------ ------
RuntimeError RuntimeError
...@@ -73,41 +68,45 @@ def generate_pcs(nni_search_space_content): ...@@ -73,41 +68,45 @@ def generate_pcs(nni_search_space_content):
""" """
categorical_dict = {} categorical_dict = {}
search_space = nni_search_space_content search_space = nni_search_space_content
def dump_categorical(fd, key, categories):
choice_len = len(categories)
if key in categorical_dict:
raise RuntimeError(
'%s has already existed, please make sure search space has no duplicate key.' % key)
categorical_dict[key] = search_space[key]['_value']
fd.write('%s categorical {%s} [0]\n' % (key, ','.join(map(str, range(choice_len)))))
with open('param_config_space.pcs', 'w') as pcs_fd: with open('param_config_space.pcs', 'w') as pcs_fd:
if isinstance(search_space, dict): if isinstance(search_space, dict):
for key in search_space.keys(): for key in search_space.keys():
if isinstance(search_space[key], dict): if isinstance(search_space[key], dict):
try: try:
if search_space[key]['_type'] == 'choice': if search_space[key]['_type'] == 'choice':
choice_len = len(search_space[key]['_value']) dump_categorical(pcs_fd, key, search_space[key]['_value'])
pcs_fd.write('%s categorical {%s} [%s]\n' % (
key,
json.dumps(list(range(choice_len)))[1:-1],
json.dumps(0)))
if key in categorical_dict:
raise RuntimeError('%s has already existed, please make sure search space has no duplicate key.' % key)
categorical_dict[key] = search_space[key]['_value']
elif search_space[key]['_type'] == 'randint': elif search_space[key]['_type'] == 'randint':
pcs_fd.write('%s integer [%d, %d] [%d]\n' % ( lower, upper = search_space[key]['_value']
key, if lower + 1 == upper:
search_space[key]['_value'][0], dump_categorical(pcs_fd, key, [lower])
search_space[key]['_value'][1] - 1, else:
search_space[key]['_value'][0])) pcs_fd.write('%s integer [%d, %d] [%d]\n' % (key, lower, upper - 1, lower))
elif search_space[key]['_type'] == 'uniform': elif search_space[key]['_type'] == 'uniform':
pcs_fd.write('%s real %s [%s]\n' % ( low, high = search_space[key]['_value']
key, if low == high:
json.dumps(search_space[key]['_value']), dump_categorical(pcs_fd, key, [low])
json.dumps(search_space[key]['_value'][0]))) else:
pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low))
elif search_space[key]['_type'] == 'loguniform': elif search_space[key]['_type'] == 'loguniform':
# use np.round here to ensure that the rounded defaut value is in the range, which will be rounded in configure_space package # use np.round here to ensure that the rounded default value is in the range,
search_space[key]['_value'] = list(np.round(np.log(search_space[key]['_value']), 10)) # which will be rounded in configure_space package
pcs_fd.write('%s real %s [%s]\n' % ( low, high = list(np.round(np.log(search_space[key]['_value']), 10))
key, if low == high:
json.dumps(search_space[key]['_value']), dump_categorical(pcs_fd, key, [search_space[key]['_value'][0]])
json.dumps(search_space[key]['_value'][0]))) else:
pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low))
elif search_space[key]['_type'] == 'quniform': elif search_space[key]['_type'] == 'quniform':
low, high, q = search_space[key]['_value'][0:3] low, high, q = search_space[key]['_value'][0:3]
vals = np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high).tolist() vals = np.clip(np.arange(np.round(low / q), np.round(high / q) + 1) * q, low, high).tolist()
pcs_fd.write('%s ordinal {%s} [%s]\n' % ( pcs_fd.write('%s ordinal {%s} [%s]\n' % (
key, key,
json.dumps(vals)[1:-1], json.dumps(vals)[1:-1],
...@@ -121,17 +120,15 @@ def generate_pcs(nni_search_space_content): ...@@ -121,17 +120,15 @@ def generate_pcs(nni_search_space_content):
return categorical_dict return categorical_dict
return None return None
def generate_scenario(ss_content): def generate_scenario(ss_content):
"""Generate the scenario. The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and """Generate the scenario. The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and
can be constructed either by providing an actual scenario-object, or by specifing the options in a scenario file. can be constructed either by providing an actual scenario-object, or by specifing the options in a scenario file.
Reference: https://automl.github.io/SMAC3/stable/options.html Reference: https://automl.github.io/SMAC3/stable/options.html
The format of the scenario file is one option per line: The format of the scenario file is one option per line:
OPTION1 = VALUE1 OPTION1 = VALUE1
OPTION2 = VALUE2 OPTION2 = VALUE2
... ...
Parameters Parameters
---------- ----------
abort_on_first_run_crash: bool abort_on_first_run_crash: bool
...@@ -194,7 +191,6 @@ def generate_scenario(ss_content): ...@@ -194,7 +191,6 @@ def generate_scenario(ss_content):
wallclock_limit: int wallclock_limit: int
Maximum amount of wallclock-time used for optimization. Default: inf. Maximum amount of wallclock-time used for optimization. Default: inf.
Use default because this is controlled by nni Use default because this is controlled by nni
Returns Returns
------- -------
Scenario: Scenario:
...@@ -203,11 +199,12 @@ def generate_scenario(ss_content): ...@@ -203,11 +199,12 @@ def generate_scenario(ss_content):
""" """
with open('scenario.txt', 'w') as sce_fd: with open('scenario.txt', 'w') as sce_fd:
sce_fd.write('deterministic = 0\n') sce_fd.write('deterministic = 0\n')
#sce_fd.write('output_dir = \n') # sce_fd.write('output_dir = \n')
sce_fd.write('paramfile = param_config_space.pcs\n') sce_fd.write('paramfile = param_config_space.pcs\n')
sce_fd.write('run_obj = quality\n') sce_fd.write('run_obj = quality\n')
return generate_pcs(ss_content) return generate_pcs(ss_content)
if __name__ == '__main__': if __name__ == '__main__':
generate_scenario('search_space.json') generate_scenario('search_space.json')
src/sdk/pynni/nni/smac_tuner/smac_tuner.py
View file @ 8c203f30
...@@ -21,34 +21,33 @@ ...@@ -21,34 +21,33 @@
smac_tuner.py smac_tuner.py
""" """
import sys
import logging import logging
import numpy as np import sys
from nni.tuner import Tuner import numpy as np
from nni.utils import OptimizeMode, extract_scalar_reward
from smac.utils.io.cmd_reader import CMDReader
from smac.scenario.scenario import Scenario
from smac.facade.smac_facade import SMAC
from smac.facade.roar_facade import ROAR
from smac.facade.epils_facade import EPILS from smac.facade.epils_facade import EPILS
from ConfigSpaceNNI import Configuration from smac.facade.roar_facade import ROAR
from smac.facade.smac_facade import SMAC
from smac.scenario.scenario import Scenario
from smac.utils.io.cmd_reader import CMDReader
from .convert_ss_to_scenario import generate_scenario from ConfigSpaceNNI import Configuration
from nni.tuner import Tuner from nni.tuner import Tuner
from nni.utils import OptimizeMode, extract_scalar_reward from nni.utils import OptimizeMode, extract_scalar_reward
from .convert_ss_to_scenario import generate_scenario
class SMACTuner(Tuner): class SMACTuner(Tuner):
""" """
Parameters Parameters
---------- ----------
optimize_mode: str optimize_mode: str
optimize mode, 'maximize' or 'minimize' optimize mode, 'maximize' or 'minimize', by default 'maximize'
""" """
def __init__(self, optimize_mode): def __init__(self, optimize_mode="maximize"):
"""Constructor""" """Constructor"""
self.logger = logging.getLogger( self.logger = logging.getLogger(
self.__module__ + "." + self.__class__.__name__) self.__module__ + "." + self.__class__.__name__)
...@@ -64,7 +63,6 @@ class SMACTuner(Tuner): ...@@ -64,7 +63,6 @@ class SMACTuner(Tuner):
def _main_cli(self): def _main_cli(self):
"""Main function of SMAC for CLI interface """Main function of SMAC for CLI interface
Returns Returns
------- -------
instance instance
...@@ -77,11 +75,9 @@ class SMACTuner(Tuner): ...@@ -77,11 +75,9 @@ class SMACTuner(Tuner):
root_logger = logging.getLogger() root_logger = logging.getLogger()
root_logger.setLevel(args.verbose_level) root_logger.setLevel(args.verbose_level)
logger_handler = logging.StreamHandler( logger_handler = logging.StreamHandler(stream=sys.stdout)
stream=sys.stdout)
if root_logger.level >= logging.INFO: if root_logger.level >= logging.INFO:
formatter = logging.Formatter( formatter = logging.Formatter("%(levelname)s:\t%(message)s")
"%(levelname)s:\t%(message)s")
else: else:
formatter = logging.Formatter( formatter = logging.Formatter(
"%(asctime)s:%(levelname)s:%(name)s:%(message)s", "%(asctime)s:%(levelname)s:%(name)s:%(message)s",
...@@ -130,15 +126,17 @@ class SMACTuner(Tuner): ...@@ -130,15 +126,17 @@ class SMACTuner(Tuner):
return optimizer return optimizer
def update_search_space(self, search_space): def update_search_space(self, search_space):
"""TODO: this is urgly, we put all the initialization work in this method, because initialization relies """
on search space, also because update_search_space is called at the beginning.
NOTE: updating search space is not supported. NOTE: updating search space is not supported.
Parameters Parameters
---------- ----------
search_space: search_space: dict
search space search space
""" """
# TODO: this is ugly, we put all the initialization work in this method, because initialization relies
# on search space, also because update_search_space is called at the beginning.
if not self.update_ss_done: if not self.update_ss_done:
self.categorical_dict = generate_scenario(search_space) self.categorical_dict = generate_scenario(search_space)
if self.categorical_dict is None: if self.categorical_dict is None:
...@@ -152,7 +150,6 @@ class SMACTuner(Tuner): ...@@ -152,7 +150,6 @@ class SMACTuner(Tuner):
def receive_trial_result(self, parameter_id, parameters, value, **kwargs): def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
"""receive_trial_result """receive_trial_result
Parameters Parameters
---------- ----------
parameter_id: int parameter_id: int
...@@ -161,7 +158,6 @@ class SMACTuner(Tuner): ...@@ -161,7 +158,6 @@ class SMACTuner(Tuner):
parameters parameters
value: value:
value value
Raises Raises
------ ------
RuntimeError RuntimeError
...@@ -179,17 +175,16 @@ class SMACTuner(Tuner): ...@@ -179,17 +175,16 @@ class SMACTuner(Tuner):
else: else:
self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward) self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward)
def convert_loguniform_categorical(self, challenger_dict): def param_postprocess(self, challenger_dict):
"""Convert the values of type `loguniform` back to their initial range """
Also, we convert categorical: Postprocessing for a set of parameter includes:
categorical values in search space are changed to list of numbers before, 1. Convert the values of type `loguniform` back to their initial range.
those original values will be changed back in this function 2. Convert categorical: categorical values in search space are changed to list of numbers before,
those original values will be changed back in this function.
Parameters Parameters
---------- ----------
challenger_dict: dict challenger_dict: dict
challenger dict challenger dict
Returns Returns
------- -------
dict dict
...@@ -210,12 +205,10 @@ class SMACTuner(Tuner): ...@@ -210,12 +205,10 @@ class SMACTuner(Tuner):
def generate_parameters(self, parameter_id, **kwargs): def generate_parameters(self, parameter_id, **kwargs):
"""generate one instance of hyperparameters """generate one instance of hyperparameters
Parameters Parameters
---------- ----------
parameter_id: int parameter_id: int
parameter id parameter id
Returns Returns
------- -------
list list
...@@ -224,21 +217,19 @@ class SMACTuner(Tuner): ...@@ -224,21 +217,19 @@ class SMACTuner(Tuner):
if self.first_one: if self.first_one:
init_challenger = self.smbo_solver.nni_smac_start() init_challenger = self.smbo_solver.nni_smac_start()
self.total_data[parameter_id] = init_challenger self.total_data[parameter_id] = init_challenger
return self.convert_loguniform_categorical(init_challenger.get_dictionary()) return self.param_postprocess(init_challenger.get_dictionary())
else: else:
challengers = self.smbo_solver.nni_smac_request_challengers() challengers = self.smbo_solver.nni_smac_request_challengers()
for challenger in challengers: for challenger in challengers:
self.total_data[parameter_id] = challenger self.total_data[parameter_id] = challenger
return self.convert_loguniform_categorical(challenger.get_dictionary()) return self.param_postprocess(challenger.get_dictionary())
def generate_multiple_parameters(self, parameter_id_list, **kwargs): def generate_multiple_parameters(self, parameter_id_list, **kwargs):
"""generate mutiple instances of hyperparameters """generate mutiple instances of hyperparameters
Parameters Parameters
---------- ----------
parameter_id_list: list parameter_id_list: list
list of parameter id list of parameter id
Returns Returns
------- -------
list list
...@@ -249,7 +240,7 @@ class SMACTuner(Tuner): ...@@ -249,7 +240,7 @@ class SMACTuner(Tuner):
for one_id in parameter_id_list: for one_id in parameter_id_list:
init_challenger = self.smbo_solver.nni_smac_start() init_challenger = self.smbo_solver.nni_smac_start()
self.total_data[one_id] = init_challenger self.total_data[one_id] = init_challenger
params.append(self.convert_loguniform_categorical(init_challenger.get_dictionary())) params.append(self.param_postprocess(init_challenger.get_dictionary()))
else: else:
challengers = self.smbo_solver.nni_smac_request_challengers() challengers = self.smbo_solver.nni_smac_request_challengers()
cnt = 0 cnt = 0
...@@ -258,16 +249,17 @@ class SMACTuner(Tuner): ...@@ -258,16 +249,17 @@ class SMACTuner(Tuner):
if cnt >= len(parameter_id_list): if cnt >= len(parameter_id_list):
break break
self.total_data[parameter_id_list[cnt]] = challenger self.total_data[parameter_id_list[cnt]] = challenger
params.append(self.convert_loguniform_categorical(challenger.get_dictionary())) params.append(self.param_postprocess(challenger.get_dictionary()))
cnt += 1 cnt += 1
return params return params
def import_data(self, data): def import_data(self, data):
"""Import additional data for tuning """
Import additional data for tuning
Parameters Parameters
---------- ----------
data: data: list of dict
a list of dictionarys, each of which has at least two keys, 'parameter' and 'value' Each of which has at least two keys, `parameter` and `value`.
""" """
_completed_num = 0 _completed_num = 0
for trial_info in data: for trial_info in data:
......
src/sdk/pynni/nni/smartparam.py
View file @ 8c203f30
...@@ -43,8 +43,6 @@ __all__ = [ ...@@ -43,8 +43,6 @@ __all__ = [
] ]
# pylint: disable=unused-argument
if trial_env_vars.NNI_PLATFORM is None: if trial_env_vars.NNI_PLATFORM is None:
def choice(*options, name=None): def choice(*options, name=None):
return param_exp.choice(options, np.random.RandomState()) return param_exp.choice(options, np.random.RandomState())
...@@ -150,42 +148,16 @@ else: ...@@ -150,42 +148,16 @@ else:
optional_input_size: number of candidate inputs to be chosen optional_input_size: number of candidate inputs to be chosen
tf: tensorflow module tf: tensorflow module
''' '''
args = (mutable_id, mutable_layer_id, funcs, funcs_args, fixed_inputs, optional_inputs, optional_input_size)
if mode == 'classic_mode': if mode == 'classic_mode':
return classic_mode(mutable_id, return classic_mode(*args)
mutable_layer_id,
funcs,
funcs_args,
fixed_inputs,
optional_inputs,
optional_input_size)
assert tf is not None, 'Internal Error: Tensorflow should not be None in modes other than classic_mode' assert tf is not None, 'Internal Error: Tensorflow should not be None in modes other than classic_mode'
if mode == 'enas_mode': if mode == 'enas_mode':
return enas_mode(mutable_id, return enas_mode(*args, tf)
mutable_layer_id,
funcs,
funcs_args,
fixed_inputs,
optional_inputs,
optional_input_size,
tf)
if mode == 'oneshot_mode': if mode == 'oneshot_mode':
return oneshot_mode(mutable_id, return oneshot_mode(*args, tf)
mutable_layer_id,
funcs,
funcs_args,
fixed_inputs,
optional_inputs,
optional_input_size,
tf)
if mode == 'darts_mode': if mode == 'darts_mode':
return darts_mode(mutable_id, return darts_mode(*args, tf)
mutable_layer_id,
funcs,
funcs_args,
fixed_inputs,
optional_inputs,
optional_input_size,
tf)
raise RuntimeError('Unrecognized mode: %s' % mode) raise RuntimeError('Unrecognized mode: %s' % mode)
def _get_param(key): def _get_param(key):
......
src/sdk/pynni/nni/trial.py
View file @ 8c203f30
...@@ -96,7 +96,7 @@ def report_final_result(metric): ...@@ -96,7 +96,7 @@ def report_final_result(metric):
'parameter_id': _params['parameter_id'], 'parameter_id': _params['parameter_id'],
'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID, 'trial_job_id': trial_env_vars.NNI_TRIAL_JOB_ID,
'type': 'FINAL', 'type': 'FINAL',
'sequence': 0, # TODO: may be unnecessary 'sequence': 0,
'value': metric 'value': metric
}) })
platform.send_metric(metric) platform.send_metric(metric)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment