SMAC tuner bug fixes (#1644)

src/sdk/pynni/nni/smac_tuner/__init__.py

+from .smac_tuner import SMACTuner
\ No newline at end of file

src/sdk/pynni/nni/smac_tuner/convert_ss_to_scenario.py

@@ -18,18 +18,17 @@
 # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-import os
 import json
+
 import numpy as np
 
+
 def get_json_content(file_path):
     """Load json file content
-
     Parameters
     ----------
     file_path:
         path to the file
-
     Raises
     ------
     TypeError
@@ -42,10 +41,10 @@ def get_json_content(file_path):
         print('Error: ', err)
         return None
 
+
 def generate_pcs(nni_search_space_content):
     """Generate the Parameter Configuration Space (PCS) which defines the
     legal ranges of the parameters to be optimized and their default values.
-
     Generally, the format is:
     # parameter_name categorical {value_1, ..., value_N} [default value]
     # parameter_name ordinal {value_1, ..., value_N} [default value]
@@ -53,19 +52,15 @@ def generate_pcs(nni_search_space_content):
     # parameter_name integer [min_value, max_value] [default value] log
     # parameter_name real [min_value, max_value] [default value]
     # parameter_name real [min_value, max_value] [default value] log
-
     Reference: https://automl.github.io/SMAC3/stable/options.html
-
     Parameters
     ----------
     nni_search_space_content: search_space
         The search space in this experiment in nni
-
     Returns
     -------
     Parameter Configuration Space (PCS)
         the legal ranges of the parameters to be optimized and their default values
-
     Raises
     ------
     RuntimeError
@@ -73,41 +68,45 @@ def generate_pcs(nni_search_space_content):
     """
     categorical_dict = {}
     search_space = nni_search_space_content
+
+    def dump_categorical(fd, key, categories):
+        choice_len = len(categories)
+        if key in categorical_dict:
+            raise RuntimeError(
+                '%s has already existed, please make sure search space has no duplicate key.' % key)
+        categorical_dict[key] = search_space[key]['_value']
+        fd.write('%s categorical {%s} [0]\n' % (key, ','.join(map(str, range(choice_len)))))
+
     with open('param_config_space.pcs', 'w') as pcs_fd:
         if isinstance(search_space, dict):
             for key in search_space.keys():
                 if isinstance(search_space[key], dict):
                     try:
                         if search_space[key]['_type'] == 'choice':
-                            choice_len = len(search_space[key]['_value'])
-                            pcs_fd.write('%s categorical {%s} [%s]\n' % (
-                                key,
-                                json.dumps(list(range(choice_len)))[1:-1],
-                                json.dumps(0)))
-                            if key in categorical_dict:
-                                raise RuntimeError('%s has already existed, please make sure search space has no duplicate key.' % key)
-                            categorical_dict[key] = search_space[key]['_value']
+                            dump_categorical(pcs_fd, key, search_space[key]['_value'])
                         elif search_space[key]['_type'] == 'randint':
-                            pcs_fd.write('%s integer [%d, %d] [%d]\n' % (
-                                key,
-                                search_space[key]['_value'][0],
-                                search_space[key]['_value'][1] - 1,
-                                search_space[key]['_value'][0]))
+                            lower, upper = search_space[key]['_value']
+                            if lower + 1 == upper:
+                                dump_categorical(pcs_fd, key, [lower])
+                            else:
+                                pcs_fd.write('%s integer [%d, %d] [%d]\n' % (key, lower, upper - 1, lower))
                         elif search_space[key]['_type'] == 'uniform':
-                            pcs_fd.write('%s real %s [%s]\n' % (
-                                key,
-                                json.dumps(search_space[key]['_value']),
-                                json.dumps(search_space[key]['_value'][0])))
+                            low, high = search_space[key]['_value']
+                            if low == high:
+                                dump_categorical(pcs_fd, key, [low])
+                            else:
+                                pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low))
                         elif search_space[key]['_type'] == 'loguniform':
-                            # use np.round here to ensure that the rounded defaut value is in the range, which will be rounded in configure_space package
-                            search_space[key]['_value'] = list(np.round(np.log(search_space[key]['_value']), 10))
-                            pcs_fd.write('%s real %s [%s]\n' % (
-                                key,
-                                json.dumps(search_space[key]['_value']),
-                                json.dumps(search_space[key]['_value'][0])))
+                            # use np.round here to ensure that the rounded default value is in the range,
+                            # which will be rounded in configure_space package
+                            low, high = list(np.round(np.log(search_space[key]['_value']), 10))
+                            if low == high:
+                                dump_categorical(pcs_fd, key, [search_space[key]['_value'][0]])
+                            else:
+                                pcs_fd.write('%s real [%s, %s] [%s]\n' % (key, low, high, low))
                         elif search_space[key]['_type'] == 'quniform':
                             low, high, q = search_space[key]['_value'][0:3]
-                            vals = np.clip(np.arange(np.round(low/q), np.round(high/q)+1) * q, low, high).tolist()
+                            vals = np.clip(np.arange(np.round(low / q), np.round(high / q) + 1) * q, low, high).tolist()
                             pcs_fd.write('%s ordinal {%s} [%s]\n' % (
                                 key,
                                 json.dumps(vals)[1:-1],
@@ -121,17 +120,15 @@ def generate_pcs(nni_search_space_content):
         return categorical_dict
     return None
 
+
 def generate_scenario(ss_content):
     """Generate the scenario. The scenario-object (smac.scenario.scenario.Scenario) is used to configure SMAC and
     can be constructed either by providing an actual scenario-object, or by specifing the options in a scenario file.
-
     Reference: https://automl.github.io/SMAC3/stable/options.html
-
     The format of the scenario file is one option per line:
     OPTION1 = VALUE1
     OPTION2 = VALUE2
     ...
-
     Parameters
     ----------
     abort_on_first_run_crash: bool
@@ -194,7 +191,6 @@ def generate_scenario(ss_content):
     wallclock_limit: int
         Maximum amount of wallclock-time used for optimization. Default: inf.
         Use default because this is controlled by nni
-
     Returns
     -------
     Scenario:
@@ -203,11 +199,12 @@ def generate_scenario(ss_content):
     """
     with open('scenario.txt', 'w') as sce_fd:
         sce_fd.write('deterministic = 0\n')
-        #sce_fd.write('output_dir = \n')
+        # sce_fd.write('output_dir = \n')
         sce_fd.write('paramfile = param_config_space.pcs\n')
         sce_fd.write('run_obj = quality\n')
 
     return generate_pcs(ss_content)
 
+
 if __name__ == '__main__':
     generate_scenario('search_space.json')

src/sdk/pynni/nni/smac_tuner/smac_tuner.py

@@ -21,34 +21,30 @@
 smac_tuner.py
 """
 
-import sys
 import logging
-import numpy as np
+import sys
 
+import numpy as np
+from ConfigSpaceNNI import Configuration
 from nni.tuner import Tuner
 from nni.utils import OptimizeMode, extract_scalar_reward
-
-from smac.utils.io.cmd_reader import CMDReader
-from smac.scenario.scenario import Scenario
-from smac.facade.smac_facade import SMAC
-from smac.facade.roar_facade import ROAR
 from smac.facade.epils_facade import EPILS
-from ConfigSpaceNNI import Configuration
+from smac.facade.roar_facade import ROAR
+from smac.facade.smac_facade import SMAC
+from smac.scenario.scenario import Scenario
+from smac.utils.io.cmd_reader import CMDReader
 
 from .convert_ss_to_scenario import generate_scenario
 
-from nni.tuner import Tuner
-from nni.utils import OptimizeMode, extract_scalar_reward
-
 
 class SMACTuner(Tuner):
     """
     Parameters
     ----------
     optimize_mode: str
-        optimize mode, 'maximize' or 'minimize'
+        optimize mode, 'maximize' or 'minimize', by default 'maximize'
     """
-    def __init__(self, optimize_mode):
+    def __init__(self, optimize_mode="maximize"):
         """Constructor"""
         self.logger = logging.getLogger(
             self.__module__ + "." + self.__class__.__name__)
@@ -64,7 +60,6 @@ class SMACTuner(Tuner):
 
     def _main_cli(self):
         """Main function of SMAC for CLI interface
-
         Returns
         -------
         instance
@@ -130,15 +125,17 @@ class SMACTuner(Tuner):
         return optimizer
 
     def update_search_space(self, search_space):
-        """TODO: this is urgly, we put all the initialization work in this method, because initialization relies
-        on search space, also because update_search_space is called at the beginning.
+        """
         NOTE: updating search space is not supported.
-
         Parameters
         ----------
-        search_space:
+        search_space: dict
             search space
         """
+
+        # TODO: this is ugly, we put all the initialization work in this method, because initialization relies
+        #         on search space, also because update_search_space is called at the beginning.
+
         if not self.update_ss_done:
             self.categorical_dict = generate_scenario(search_space)
             if self.categorical_dict is None:
@@ -152,7 +149,6 @@ class SMACTuner(Tuner):
 
     def receive_trial_result(self, parameter_id, parameters, value, **kwargs):
         """receive_trial_result
-
         Parameters
         ----------
         parameter_id: int
@@ -161,7 +157,6 @@ class SMACTuner(Tuner):
             parameters
         value:
             value
-
         Raises
         ------
         RuntimeError
@@ -179,17 +174,16 @@ class SMACTuner(Tuner):
         else:
             self.smbo_solver.nni_smac_receive_runs(self.total_data[parameter_id], reward)
 
-    def convert_loguniform_categorical(self, challenger_dict):
-        """Convert the values of type `loguniform` back to their initial range
-        Also, we convert categorical:
-        categorical values in search space are changed to list of numbers before,
-        those original values will be changed back in this function
-
+    def param_postprocess(self, challenger_dict):
+        """
+        Postprocessing for a set of parameter includes:
+        1. Convert the values of type `loguniform` back to their initial range.
+        2. Convert categorical: categorical values in search space are changed to list of numbers before,
+        those original values will be changed back in this function.
         Parameters
         ----------
         challenger_dict: dict
             challenger dict
-
         Returns
         -------
         dict
@@ -210,12 +204,10 @@ class SMACTuner(Tuner):
 
     def generate_parameters(self, parameter_id, **kwargs):
         """generate one instance of hyperparameters
-
         Parameters
         ----------
         parameter_id: int
             parameter id
-
         Returns
         -------
         list
@@ -224,21 +216,19 @@ class SMACTuner(Tuner):
         if self.first_one:
             init_challenger = self.smbo_solver.nni_smac_start()
             self.total_data[parameter_id] = init_challenger
-            return self.convert_loguniform_categorical(init_challenger.get_dictionary())
+            return self.param_postprocess(init_challenger.get_dictionary())
         else:
             challengers = self.smbo_solver.nni_smac_request_challengers()
             for challenger in challengers:
                 self.total_data[parameter_id] = challenger
-                return self.convert_loguniform_categorical(challenger.get_dictionary())
+                return self.param_postprocess(challenger.get_dictionary())
 
     def generate_multiple_parameters(self, parameter_id_list, **kwargs):
         """generate mutiple instances of hyperparameters
-
         Parameters
         ----------
         parameter_id_list: list
             list of parameter id
-
         Returns
         -------
         list
@@ -249,7 +239,7 @@ class SMACTuner(Tuner):
             for one_id in parameter_id_list:
                 init_challenger = self.smbo_solver.nni_smac_start()
                 self.total_data[one_id] = init_challenger
-                params.append(self.convert_loguniform_categorical(init_challenger.get_dictionary()))
+                params.append(self.param_postprocess(init_challenger.get_dictionary()))
         else:
             challengers = self.smbo_solver.nni_smac_request_challengers()
             cnt = 0
@@ -258,16 +248,17 @@ class SMACTuner(Tuner):
                 if cnt >= len(parameter_id_list):
                     break
                 self.total_data[parameter_id_list[cnt]] = challenger
-                params.append(self.convert_loguniform_categorical(challenger.get_dictionary()))
+                params.append(self.param_postprocess(challenger.get_dictionary()))
                 cnt += 1
         return params
 
     def import_data(self, data):
-        """Import additional data for tuning
+        """
+        Import additional data for tuning
         Parameters
         ----------
-        data:
-            a list of dictionarys, each of which has at least two keys, 'parameter' and 'value'
+        data: list of dict
+            Each of which has at least two keys, `parameter` and `value`.
         """
         _completed_num = 0
         for trial_info in data: