Refactor Hyperopt Tuners (Stage 1) - random tuner (#4118)

nni/algorithms/hpo/random_tuner.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+import numpy as np
+import schema
+
+from nni import ClassArgsValidator
+from nni.common.hpo_utils import format_search_space, deformat_parameters
+from nni.tuner import Tuner
+
+class RandomTuner(Tuner):
+    def __init__(self, seed=None):
+        self.space = None
+        self.rng = np.random.default_rng(seed)
+
+    def update_search_space(self, space):
+        self.space = format_search_space(space)
+
+    def generate_parameters(self, *args, **kwargs):
+        params = suggest(self.rng, self.space)
+        return deformat_parameters(params, self.space)
+
+    def receive_trial_result(self, *args, **kwargs):
+        pass
+
+class RandomClassArgsValidator(ClassArgsValidator):
+    def validate_class_args(self, **kwargs):
+        schema.Schema({schema.Optional('seed'): int}).validate(kwargs)
+
+def suggest(rng, space):
+    params = {}
+    for spec in space.values():
+        if not spec.is_activated(params):
+            continue
+
+        if spec.categorical:
+            params[spec.key] = rng.integers(spec.size)
+            continue
+
+        if spec.normal_distributed:
+            if spec.log_distributed:
+                x = rng.lognormal(spec.mu, spec.sigma)
+            else:
+                x = rng.normal(spec.mu, spec.sigma)
+        else:
+            if spec.log_distributed:
+                x = np.exp(rng.uniform(np.log(spec.low), np.log(spec.high)))
+            else:
+                x = rng.uniform(spec.low, spec.high)
+        if spec.q is not None:
+            x = np.round(x / spec.q) * spec.q
+        params[spec.key] = x
+    return params

nni/common/hpo_utils/__init__.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+from .validation import validate_search_space
+from .formatting import *

nni/common/hpo_utils/formatting.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""
+This script provides a more program-friendly representation of HPO search space.
+The format is considered internal helper and is not visible to end users.
+
+You will find this useful when you want to support nested search space.
+"""
+
+__all__ = [
+    'ParameterSpec',
+    'deformat_parameters',
+    'format_search_space',
+]
+
+import math
+from typing import Any, List, NamedTuple, Optional, Tuple
+
+class ParameterSpec(NamedTuple):
+    """
+    Specification (aka space / range) of one single parameter.
+    """
+
+    name: str                       # The object key in JSON
+    type: str                       # "_type" in JSON
+    values: List[Any]               # "_value" in JSON
+
+    key: Tuple[str]                 # The "path" of this parameter
+    parent_index: Optional[int]     # If the parameter is in a nested choice, this is its parent's index;
+                                    # if the parameter is at top level, this is `None`.
+
+    categorical: bool               # Whether this paramter is categorical (unordered) or numerical (ordered)
+    size: int = None                # If it's categorical, how many canidiates it has
+
+    # uniform distributed
+    low: float = None               # Lower bound of uniform parameter
+    high: float = None              # Upper bound of uniform parameter
+
+    normal_distributed: bool = None # Whether this parameter is uniform or normal distrubuted
+    mu: float = None                # Mean of normal parameter
+    sigma: float = None             # Scale of normal parameter
+
+    q: Optional[float] = None       # If not `None`, the value should be an integer multiple of this
+    log_distributed: bool = None    # Whether this parameter is log distributed
+
+    def is_activated(self, partial_parameters):
+        """
+        For nested search space, check whether this parameter should be skipped for current set of paremters.
+        This function works because the return value of `format_search_space()` is sorted in a way that
+        parents always appear before children.
+        """
+        return self.parent_index is None or partial_parameters.get(self.key[:-1]) == self.parent_index
+
+def format_search_space(search_space, ordered_randint=False):
+    formatted = _format_search_space(tuple(), None, search_space)
+    if ordered_randint:
+        for i, spec in enumerate(formatted):
+            if spec.type == 'randint':
+                formatted[i] = _format_ordered_randint(spec.key, spec.parent_index, spec.values)
+    return {spec.key: spec for spec in formatted}
+
+def deformat_parameters(parameters, formatted_search_space):
+    """
+    `paramters` is a dict whose key is `ParamterSpec.key`, and value is integer index if the parameter is categorical.
+    Convert it to the format expected by end users.
+    """
+    ret = {}
+    for key, x in parameters.items():
+        spec = formatted_search_space[key]
+        if not spec.categorical:
+            _assign(ret, key, x)
+        elif spec.type == 'randint':
+            lower = min(math.ceil(float(x)) for x in spec.values)
+            _assign(ret, key, lower + x)
+        elif _is_nested_choices(spec.values):
+            _assign(ret, tuple([*key, '_name']), spec.values[x]['_name'])
+        else:
+            _assign(ret, key, spec.values[x])
+    return ret
+
+def _format_search_space(parent_key, parent_index, space):
+    formatted = []
+    for name, spec in space.items():
+        if name == '_name':
+            continue
+        key = tuple([*parent_key, name])
+        formatted.append(_format_parameter(key, parent_index, spec['_type'], spec['_value']))
+        if spec['_type'] == 'choice' and _is_nested_choices(spec['_value']):
+            for index, sub_space in enumerate(spec['_value']):
+                formatted += _format_search_space(key, index, sub_space)
+    return formatted
+
+def _format_parameter(key, parent_index, type_, values):
+    spec = {}
+    spec['name'] = key[-1]
+    spec['type'] = type_
+    spec['values'] = values
+
+    spec['key'] = key
+    spec['parent_index'] = parent_index
+
+    if type_ in ['choice', 'randint']:
+        spec['categorical'] = True
+        if type_ == 'choice':
+            spec['size'] = len(values)
+        else:
+            lower, upper = sorted(math.ceil(float(x)) for x in values)
+            spec['size'] = upper - lower
+
+    else:
+        spec['categorical'] = False
+        if type_.startswith('q'):
+            spec['q'] = float(values[2])
+        spec['log_distributed'] = ('log' in type_)
+
+        if 'normal' in type_:
+            spec['normal_distributed'] = True
+            spec['mu'] = float(values[0])
+            spec['sigma'] = float(values[1])
+
+        else:
+            spec['normal_distributed'] = False
+            spec['low'], spec['high'] = sorted(float(x) for x in values[:2])
+            if 'q' in spec:
+                spec['low'] = math.ceil(spec['low'] / spec['q']) * spec['q']
+                spec['high'] = math.floor(spec['high'] / spec['q']) * spec['q']
+
+    return ParameterSpec(**spec)
+
+def _format_ordered_randint(key, parent_index, values):
+    lower, upper = sorted(math.ceil(float(x)) for x in values)
+    return ParameterSpec(
+        name = key[-1],
+        type = 'randint',
+        values = values,
+        key = key,
+        parent_index = parent_index,
+        categorical = False,
+        low = float(lower),
+        high = float(upper - 1),
+        normal_distributed = False,
+        q = 1.0,
+        log_distributed = False,
+    )
+
+def _is_nested_choices(values):
+    if not values:
+        return False
+    for value in values:
+        if not isinstance(value, dict):
+            return False
+        if '_name' not in value:
+            return False
+    return True
+
+def _assign(params, key, x):
+    if len(key) == 1:
+        params[key[0]] = x
+    else:
+        if key[0] not in params:
+            params[key[0]] = {}
+        _assign(params[key[0]], key[1:], x)

nni/common/hpo_utils.py → nni/common/hpo_utils/validation.py

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
 import logging
 from typing import Any, List, Optional
 

nni/runtime/default_config/registered_algorithms.yml

@@ -31,12 +31,9 @@ tuners:
   classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.HyperoptClassArgsValidator
   className: nni.algorithms.hpo.hyperopt_tuner.HyperoptTuner
   source: nni
-- acceptClassArgs: false
-  builtinName: Random
-  classArgs:
-    algorithm_name: random_search
-  classArgsValidator: nni.algorithms.hpo.hyperopt_tuner.HyperoptClassArgsValidator
-  className: nni.algorithms.hpo.hyperopt_tuner.HyperoptTuner
+- builtinName: Random
+  className: nni.algorithms.hpo.random_tuner.RandomTuner
+  classArgsValidator: nni.algorithms.hpo.random_tuner.RandomClassArgsValidator
   source: nni
 - builtinName: Anneal
   classArgs:

test/ut/sdk/test_hpo_format_space.py

+from nni.common.hpo_utils import format_search_space, deformat_parameters
+
+user_space = {
+    'dropout_rate': { '_type': 'uniform', '_value': [0.5, 0.9] },
+    'conv_size': { '_type': 'choice', '_value': [2, 3, 5, 7] },
+    'hidden_size': { '_type': 'qloguniform', '_value': [128, 1024, 1] },
+    'batch_size': { '_type': 'randint', '_value': [16, 32] },
+    'learning_rate': { '_type': 'loguniform', '_value': [0.0001, 0.1] },
+    'nested': {
+        '_type': 'choice',
+        '_value': [
+            {
+                '_name': 'empty',
+            },
+            {
+                '_name': 'double_nested',
+                'xy': {
+                    '_type': 'choice',
+                    '_value': [
+                        {
+                            '_name': 'x',
+                            'x': { '_type': 'normal', '_value': [0, 1.0] },
+                        },
+                        {
+                            '_name': 'y',
+                            'y': { '_type': 'qnormal', '_value': [0, 1, 0.5] },
+                        },
+                    ],
+                },
+                'z': { '_type': 'quniform', '_value': [-0.5, 0.5, 0.1] },
+            },
+            {
+                '_name': 'common',
+                'x': { '_type': 'lognormal', '_value': [1, 0.1] },
+                'y': { '_type': 'qlognormal', '_value': [-1, 1, 0.1] },
+            },
+        ],
+    },
+}
+
+internal_space_simple = [  # the full internal space is too long, omit None and False values here
+    {'name':'dropout_rate', 'type':'uniform', 'values':[0.5,0.9], 'key':('dropout_rate',), 'low':0.5, 'high':0.9},
+    {'name':'conv_size', 'type':'choice', 'values':[2,3,5,7], 'key':('conv_size',), 'categorical':True, 'size':4},
+    {'name':'hidden_size', 'type':'qloguniform', 'values':[128,1024,1], 'key':('hidden_size',), 'low':128.0, 'high':1024.0, 'q':1.0, 'log_distributed':True},
+    {'name':'batch_size', 'type':'randint', 'values':[16,32], 'key':('batch_size',), 'categorical':True, 'size':16},
+    {'name':'learning_rate', 'type':'loguniform', 'values':[0.0001,0.1], 'key':('learning_rate',), 'low':0.0001, 'high':0.1, 'log_distributed':True},
+    {'name':'nested', 'type':'choice', '_value_names':['empty','double_nested','common'], 'key':('nested',), 'categorical':True, 'size':3, 'nested_choice':True},
+    {'name':'xy', 'type':'choice', '_value_names':['x','y'], 'key':('nested','xy'), 'parent_index':1, 'categorical':True, 'size':2, 'nested_choice':True},
+    {'name':'x', 'type':'normal', 'values':[0,1.0], 'key':('nested','xy','x'), 'parent_index':0, 'normal_distributed':True, 'mu':0.0, 'sigma':1.0},
+    {'name':'y', 'type':'qnormal', 'values':[0,1,0.5], 'key':('nested','xy','y'), 'parent_index':1, 'normal_distributed':True, 'mu':0.0, 'sigma':1.0, 'q':0.5},
+    {'name':'z', 'type':'quniform', 'values':[-0.5,0.5,0.1], 'key':('nested','z'), 'parent_index':1, 'low':-0.5, 'high':0.5, 'q':0.1},
+    {'name':'x', 'type':'lognormal', 'values':[1,0.1], 'key':('nested','x'), 'parent_index':2, 'normal_distributed':True, 'mu':1.0, 'sigma':0.1, 'log_distributed':True},
+    {'name':'y', 'type':'qlognormal', 'values':[-1,1,0.1], 'key':('nested','y'), 'parent_index':2, 'normal_distributed':True, 'mu':-1.0, 'sigma':1.0, 'q':0.1, 'log_distributed':True},
+]
+
+def test_format_search_space():
+    formatted = format_search_space(user_space)
+    for spec, expected in zip(formatted.values(), internal_space_simple):
+        for key, value in spec._asdict().items():
+            if key == 'values' and '_value_names' in expected:
+                assert [v['_name'] for v in value] == expected['_value_names']
+            elif key in expected:
+                assert value == expected[key]
+            else:
+                assert value is None or value == False
+
+internal_parameters = {
+    ('dropout_rate',): 0.7,
+    ('conv_size',): 2,
+    ('hidden_size',): 200.0,
+    ('batch_size',): 3,
+    ('learning_rate',): 0.0345,
+    ('nested',): 1,
+    ('nested', 'xy'): 0,
+    ('nested', 'xy', 'x'): 0.123,
+}
+
+user_parameters = {
+    'dropout_rate': 0.7,
+    'conv_size': 5,
+    'hidden_size': 200.0,
+    'batch_size': 19,
+    'learning_rate': 0.0345,
+    'nested': {
+        '_name': 'double_nested',
+        'xy': {
+            '_name': 'x',
+            'x': 0.123,
+        },
+    },
+}
+
+def test_deformat_parameters():
+    space = format_search_space(user_space)
+    generated = deformat_parameters(internal_parameters, space)
+    assert generated == user_parameters
+
+if __name__ == '__main__':
+    test_format_search_space()
+    test_deformat_parameters()